diff --git a/backend-compliance/compliance/api/agent_compliance_check_routes.py b/backend-compliance/compliance/api/agent_compliance_check_routes.py
index 3a62c3a5..0073ac42 100644
--- a/backend-compliance/compliance/api/agent_compliance_check_routes.py
+++ b/backend-compliance/compliance/api/agent_compliance_check_routes.py
@@ -9,6 +9,7 @@ GET /compliance/agent/compliance-check/{check_id} — poll status
import asyncio
import logging
import os
+import re
import uuid as _uuid
from dataclasses import asdict
from datetime import datetime, timezone
@@ -600,32 +601,65 @@ async def _autodiscover_missing(
})
return
- base = max(bases, key=bases.get) + "/"
+ # Build crawl plan: primary base + any related domains mentioned in
+ # the submitted texts that share the owner's SLD. Example: BMW Group
+ # text mentions bmwgroup.com and bmwgroup.jobs in addition to bmw.de.
+ primary_base = max(bases, key=bases.get) + "/"
+ crawl_bases: list[str] = [primary_base]
+ primary_netloc = urlparse(primary_base).netloc.lower().lstrip("www.")
+ owner_token = primary_netloc.split(".")[0] # 'bmw'
+
+ if owner_token and len(owner_token) >= 3:
+ domain_re = re.compile(
+ r"https?://([a-z0-9][a-z0-9\-]*\.)*" + re.escape(owner_token)
+ + r"[a-z0-9\-]*\.[a-z]{2,}",
+ re.IGNORECASE,
+ )
+ seen_bases = {primary_base}
+ for entry in doc_entries:
+ text = entry.get("text") or ""
+ for m in domain_re.finditer(text):
+ p = urlparse(m.group(0))
+ base = f"{p.scheme}://{p.netloc}/"
+ base_netloc = p.netloc.lower().lstrip("www.")
+ if base_netloc == primary_netloc:
+ continue
+ if base in seen_bases:
+ continue
+ seen_bases.add(base)
+ crawl_bases.append(base)
+ if len(crawl_bases) >= 3:
+ break
+ if len(crawl_bases) >= 3:
+ break
+
_update(
check_id,
- f"Suche fehlende Dokumente auf {urlparse(base).netloc}...",
+ f"Suche fehlende Dokumente auf {', '.join(urlparse(b).netloc for b in crawl_bases)}...",
18,
)
- try:
- async with httpx.AsyncClient(timeout=180.0) as client:
- resp = await client.post(
- f"{CONSENT_TESTER_URL}/dsi-discovery",
- json={"url": base, "max_documents": 15},
- timeout=180.0,
- )
- if resp.status_code != 200:
- logger.warning("auto-discovery: HTTP %d for %s", resp.status_code, base)
- discovered: list[dict] = []
- disc_payloads: list[dict] = []
- else:
- disc_body = resp.json()
- discovered = disc_body.get("documents", [])
- disc_payloads = disc_body.get("cmp_payloads") or []
- except Exception as e:
- logger.warning("auto-discovery failed for %s: %s", base, e)
- discovered = []
- disc_payloads = []
+ discovered: list[dict] = []
+ disc_payloads: list[dict] = []
+ for base in crawl_bases:
+ try:
+ async with httpx.AsyncClient(timeout=180.0) as client:
+ resp = await client.post(
+ f"{CONSENT_TESTER_URL}/dsi-discovery",
+ json={"url": base, "max_documents": 15},
+ timeout=180.0,
+ )
+ if resp.status_code != 200:
+ logger.warning("auto-discovery: HTTP %d for %s",
+ resp.status_code, base)
+ continue
+ body = resp.json()
+ discovered.extend(body.get("documents", []) or [])
+ disc_payloads.extend(body.get("cmp_payloads") or [])
+ logger.info("auto-discovery on %s: %d docs",
+ base, len(body.get("documents", []) or []))
+ except Exception as e:
+ logger.warning("auto-discovery failed for %s: %s", base, e)
# Classify each discovered doc into a canonical doc_type
by_type: dict[str, dict] = {}
@@ -736,8 +770,12 @@ async def _check_single(
# Master Control checks (top 20 by severity to avoid noise)
try:
+ # max_controls=0 -> evaluate ALL MCs for this doc_type (DB has
+ # 1874 across 8 types; regex matching is cheap and dominates
+ # well under 1s per doc). Caps remain on the LLM-enrich step
+ # (top-10 FAILs) so cost stays bounded.
mc_results = await check_document_with_controls(
- text, doc_type, label, max_controls=20, use_agent=use_agent,
+ text, doc_type, label, max_controls=0, use_agent=use_agent,
)
if mc_results:
for mc in mc_results:
diff --git a/backend-compliance/compliance/services/doc_checks/llm_verify.py b/backend-compliance/compliance/services/doc_checks/llm_verify.py
index 57d53a1f..c14f71b8 100644
--- a/backend-compliance/compliance/services/doc_checks/llm_verify.py
+++ b/backend-compliance/compliance/services/doc_checks/llm_verify.py
@@ -62,7 +62,13 @@ async def verify_failed_checks(
async def _ask_llm_batch(
text: str, checks: list[dict], doc_title: str,
) -> dict[str, dict]:
- """Ask the LLM to verify ALL failed checks in a single call."""
+ """Ask the LLM to verify ALL failed checks in a single call.
+
+ Uses /api/chat with format='json' so Ollama enforces a valid JSON
+ response object — much more reliable than the previous /api/generate
+ + free-text approach which qwen3 often wrapped in ...
+ reasoning tokens.
+ """
checklist_lines = []
for i, c in enumerate(checks, 1):
checklist_lines.append(
@@ -70,69 +76,113 @@ async def _ask_llm_batch(
)
checklist_str = "\n".join(checklist_lines)
- prompt = f"""/no_think
-Pruefe ob der Dokumenttext die folgenden Anforderungen erfuellt.
-DOKUMENT: "{doc_title}"
+ system_msg = (
+ "Du pruefst ob ein Dokument bestimmte Pflichtangaben enthaelt. "
+ "Antworte AUSSCHLIESSLICH mit einem JSON-Objekt: "
+ '{"results": [{"id": "", "found": true|false, '
+ '"evidence": ""}]}. '
+ "Keine Erklaerungen, keine Reasoning-Tags, kein Markdown."
+ )
+ user_msg = (
+ f'DOKUMENT: "{doc_title}"\n\n'
+ f"ANFORDERUNGEN:\n{checklist_str}\n\n"
+ f"TEXT:\n{text}"
+ )
-ANFORDERUNGEN:
-{checklist_str}
+ payload = {
+ "model": OLLAMA_MODEL,
+ "messages": [
+ {"role": "system", "content": system_msg},
+ {"role": "user", "content": user_msg},
+ ],
+ "stream": False,
+ "format": "json", # forces valid JSON output
+ "options": {"temperature": 0.0, "num_predict": 3000},
+ }
-TEXT:
-{text}
-
-Antworte NUR mit einem JSON-Array (keine Erklaerung). Fuer jede Anforderung:
-[{{"id": "check-id", "found": true/false, "evidence": "Kurzes Zitat (max 80 Zeichen) oder leer"}}]
-"""
-
- async with httpx.AsyncClient(timeout=90.0) as client:
- resp = await client.post(
- f"{OLLAMA_URL}/api/generate",
- json={
- "model": OLLAMA_MODEL,
- "prompt": prompt,
- "stream": False,
- "options": {"temperature": 0.0, "num_predict": 2000},
- },
- )
+ async with httpx.AsyncClient(timeout=120.0) as client:
+ resp = await client.post(f"{OLLAMA_URL}/api/chat", json=payload)
resp.raise_for_status()
- raw = resp.json().get("response", "")
+ data = resp.json()
+ raw = (data.get("message") or {}).get("content", "")
return _parse_batch_response(raw, checks)
def _parse_batch_response(raw: str, checks: list[dict]) -> dict[str, dict]:
- """Parse batch LLM JSON array response."""
+ """Parse batch LLM response. Tolerates … wrappers,
+ code-fences, and either {results: [...]} or top-level [...]."""
import json
import re
results: dict[str, dict] = {}
- raw = raw.strip()
+ if not raw:
+ logger.info("LLM batch: empty response from model")
+ return results
- # Extract JSON array from markdown code blocks
- m = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", raw, re.DOTALL)
+ text = raw.strip()
+ # Strip qwen3 thinking tags
+ text = re.sub(r".*?", "", text, flags=re.DOTALL).strip()
+ # Strip markdown code fences
+ m = re.search(r"```(?:json)?\s*(.+?)\s*```", text, re.DOTALL)
if m:
- raw = m.group(1)
- else:
- m = re.search(r"\[.*\]", raw, re.DOTALL)
- if m:
- raw = m.group(0)
+ text = m.group(1).strip()
+ # Try parse as-is
+ parsed = None
try:
- items = json.loads(raw)
- if isinstance(items, list):
- for item in items:
- cid = item.get("id", "")
- if cid:
- results[cid] = {
- "found": bool(item.get("found", False)),
- "evidence": str(item.get("evidence", ""))[:150],
- }
+ parsed = json.loads(text)
except (json.JSONDecodeError, ValueError):
- # Fallback: extract individual JSON objects
- for m in re.finditer(r'\{[^}]*"id"\s*:\s*"([^"]+)"[^}]*"found"\s*:\s*(true|false)[^}]*\}', raw, re.DOTALL):
- cid = m.group(1)
- found = m.group(2) == "true"
- results[cid] = {"found": found, "evidence": ""}
+ # Try finding the first JSON object or array in the text
+ for pattern in (r"\{.*\}", r"\[.*\]"):
+ mm = re.search(pattern, text, re.DOTALL)
+ if mm:
+ try:
+ parsed = json.loads(mm.group(0))
+ break
+ except (json.JSONDecodeError, ValueError):
+ continue
+
+ if parsed is None:
+ logger.info(
+ "LLM batch: 0/%d checks parsed (raw head: %r)",
+ len(checks), raw[:120],
+ )
+ return results
+
+ # Accept both {"results": [...]} (preferred) and bare list
+ items = None
+ if isinstance(parsed, dict):
+ for key in ("results", "checks", "items", "verifications"):
+ if isinstance(parsed.get(key), list):
+ items = parsed[key]
+ break
+ elif isinstance(parsed, list):
+ items = parsed
+
+ if not items:
+ # Final fallback: regex over individual id/found pairs
+ for mm in re.finditer(
+ r'\{[^}]*"id"\s*:\s*"([^"]+)"[^}]*"found"\s*:\s*(true|false)[^}]*\}',
+ raw, re.DOTALL,
+ ):
+ results[mm.group(1)] = {
+ "found": mm.group(2) == "true", "evidence": "",
+ }
+ logger.info("LLM batch: %d/%d checks parsed (regex fallback)",
+ len(results), len(checks))
+ return results
+
+ for item in items:
+ if not isinstance(item, dict):
+ continue
+ cid = item.get("id", "")
+ if not cid:
+ continue
+ results[cid] = {
+ "found": bool(item.get("found", False)),
+ "evidence": str(item.get("evidence", ""))[:150],
+ }
logger.info("LLM batch: %d/%d checks parsed", len(results), len(checks))
return results
diff --git a/backend-compliance/compliance/services/doc_checks/loeschkonzept_checks.py b/backend-compliance/compliance/services/doc_checks/loeschkonzept_checks.py
new file mode 100644
index 00000000..9a47a5b5
--- /dev/null
+++ b/backend-compliance/compliance/services/doc_checks/loeschkonzept_checks.py
@@ -0,0 +1,191 @@
+"""
+Loeschkonzept checks — Art. 5(1)(e) DSGVO ('Speicherbegrenzung'),
+DIN 66398 (Leitlinie zur Entwicklung eines Loeschkonzepts).
+
+L1: Pflichtangabe vorhanden?
+L2: Pflichtangabe vollstaendig/korrekt?
+"""
+
+LOESCHKONZEPT_CHECKLIST = [
+ # ── L1: Geltungsbereich + Verantwortliche ─────────────────────────
+ {
+ "id": "scope_responsibility",
+ "label": "Geltungsbereich + Verantwortliche benannt",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"(?:geltungsbereich|anwendungsbereich)",
+ r"verantwortlich\w*\s+(?:fuer|für)\s+(?:das\s+)?l(?:oe|ö)schkonzept",
+ r"(?:datenschutzbeauftragt\w*|dpo|dsb)\s+(?:verantwort|zustaendig|zuständig)",
+ ],
+ "severity": "HIGH",
+ "hint": "DIN 66398 verlangt einen klaren Geltungsbereich (welche Systeme, Datenarten, Standorte) und die Benennung des Verantwortlichen fuer Erstellung + Wartung des Loeschkonzepts.",
+ },
+
+ # ── L1: Datenkategorien ───────────────────────────────────────────
+ {
+ "id": "data_categories",
+ "label": "Datenkategorien / Datenarten dokumentiert",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"(?:daten[\-\s]?kategori|datenart)\w*",
+ r"(?:kunden|mitarbeiter|interessent|bewerber|lieferant)daten",
+ r"personenbezogene?\s+daten[\-\s]?(?:art|kategori|gruppe)",
+ ],
+ "severity": "HIGH",
+ "hint": "Pro Datenkategorie (Kundenstammdaten, Vertragsdaten, Mitarbeiterdaten, Bewerber, Lieferantendaten, etc.) muss eine eigene Loeschfrist festgelegt werden.",
+ },
+ {
+ "id": "data_categories_specific",
+ "label": "Konkrete Datenarten benannt (>= 5)",
+ "level": 2, "parent": "data_categories",
+ "patterns": [
+ r"(?:stammdaten|kontaktdaten|vertragsdaten|abrechnungsdaten|"
+ r"protokoll\w+|log[\-\s]?daten|gesundheits|biometrische?|"
+ r"finanz|bewerbungsunterlagen|kreditdaten|tracking)",
+ ],
+ "severity": "MEDIUM",
+ "hint": "Beispiele aus der Praxis: Stammdaten, Kontaktdaten, Vertragsdaten, Abrechnungsdaten, Bewerbungsunterlagen, Log-Daten, Tracking-Cookies. Jeweils mit eigener Frist.",
+ },
+
+ # ── L1: Loeschfristen ─────────────────────────────────────────────
+ {
+ "id": "retention_periods",
+ "label": "Konkrete Loeschfristen festgelegt",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"l(?:oe|ö)schfrist",
+ r"aufbewahrungs(?:frist|dauer|pflicht)",
+ r"\d+\s*(?:jahr|monat|tag|woche)",
+ r"speicherdauer",
+ ],
+ "severity": "HIGH",
+ "hint": "Art. 5(1)(e) DSGVO + Art. 13(2)(a) DSGVO: Speicherdauer oder Kriterien fuer die Festlegung muessen pro Datenkategorie konkret benannt sein.",
+ },
+ {
+ "id": "legal_retention_basis",
+ "label": "Gesetzliche Aufbewahrungspflichten referenziert",
+ "level": 2, "parent": "retention_periods",
+ "patterns": [
+ r"§\s*257\s*hgb|§\s*147\s*ao|hgb\s+§\s*257|ao\s+§\s*147",
+ r"6\s+jahr\w*\s+\((?:hgb|handels)",
+ r"10\s+jahr\w*\s+\((?:ao|steuer)",
+ r"handelsrechtlich|steuerrechtlich",
+ r"§\s*195\s*bgb|verjaehrung\w*\s+\(bgb",
+ ],
+ "severity": "MEDIUM",
+ "hint": "Standardfristen referenzieren: §257 HGB (6 Jahre Buchungsbelege), §147 AO (10 Jahre Steuerunterlagen), §195 BGB (3 Jahre Verjaehrung). Ohne Referenz wirken die Fristen willkuerlich.",
+ },
+ {
+ "id": "deletion_trigger",
+ "label": "Loeschtrigger / Loeschanlass dokumentiert",
+ "level": 2, "parent": "retention_periods",
+ "patterns": [
+ r"l(?:oe|ö)sch[\-\s]?(?:trigger|anlass|grund|kriteri)",
+ r"(?:nach|bei)\s+(?:vertragsende|kuendigung|kündigung|abschluss)",
+ r"zweckwegfall|zweck\s+entfaellt",
+ r"einwilligungswiderruf",
+ ],
+ "severity": "MEDIUM",
+ "hint": "Pro Datenkategorie muss klar sein WANN die Frist beginnt: Vertragsende, Kuendigung, letzter Kontakt, Zweckwegfall, Einwilligungswiderruf. Nur 'X Jahre' ohne Startpunkt ist unscharf.",
+ },
+
+ # ── L1: Loeschprozess ─────────────────────────────────────────────
+ {
+ "id": "deletion_process",
+ "label": "Loeschprozess beschrieben",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"l(?:oe|ö)sch(?:prozess|vorgang|verfahren|workflow|routine)",
+ r"(?:wie|wann)\s+(?:wird|werden)\s+(?:die\s+daten\s+)?gel(?:oe|ö)scht",
+ r"automatisierte?\s+l(?:oe|ö)schung",
+ ],
+ "severity": "HIGH",
+ "hint": "Beschreiben wie Loeschung erfolgt: automatisch per Cron-Job, manuell durch Admin, Loeschungs-Workflow im CRM, Backup-Loeschung etc.",
+ },
+ {
+ "id": "deletion_technical",
+ "label": "Technische Loeschmethode benannt",
+ "level": 2, "parent": "deletion_process",
+ "patterns": [
+ r"(?:physisch\w*|sicher\w*)\s+l(?:oe|ö)schung",
+ r"(?:ueberschr|überschr)eiben\w*\s*(?:der|von)?\s*daten",
+ r"(?:dod[\-\s]?5220|nist[\-\s]?800|crypto[\-\s]?shredding)",
+ r"papierakten?\s*(?:vernicht|schreddern|verbrenn)",
+ r"datentraeger\w*\s+(?:zerstoer|vernicht|entmagnetis)",
+ ],
+ "severity": "LOW",
+ "hint": "Technische Standards nennen: DoD 5220.22-M (mehrfaches Ueberschreiben), NIST 800-88, Crypto-Shredding (Verschluesselung + Schluesselvernichtung), Papier per DIN 66399.",
+ },
+ {
+ "id": "deletion_systems",
+ "label": "Loeschung in allen relevanten Systemen",
+ "level": 2, "parent": "deletion_process",
+ "patterns": [
+ r"backup\w*\s+(?:l(?:oe|ö)sch|umfass|einbezogen|loeschr|löschr)",
+ r"archive?\w*\s+(?:l(?:oe|ö)sch|enthalten|inbegriffen|umfass)",
+ r"(?:crm|erp|datenbank|datawarehouse|dwh)\w*\s+l(?:oe|ö)sch",
+ r"(?:alle|saemtliche|sämtliche)\s+systeme",
+ ],
+ "severity": "LOW",
+ "hint": "Loeschung muss in ALLEN Systemen erfolgen: CRM, ERP, Backups, Archive, Data Warehouse, lokale Kopien. Backups die laenger als die Loeschfrist aufbewahrt werden sind kritisch (gerichtlich umstritten).",
+ },
+
+ # ── L1: Loeschnachweis ────────────────────────────────────────────
+ {
+ "id": "deletion_proof",
+ "label": "Loeschnachweis / Loeschprotokoll",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"l(?:oe|ö)sch[\-\s]?(?:nachweis|protokoll|dokumentation|log)",
+ r"(?:protokoll|dokument)\w*\s+(?:der|alle)\s+l(?:oe|ö)schung",
+ r"audit[\-\s]?trail.*l(?:oe|ö)sch",
+ ],
+ "severity": "MEDIUM",
+ "hint": "Art. 5(2) DSGVO (Rechenschaftspflicht): Loeschvorgaenge muessen nachweisbar sein. Mindestens: was, wann, durch wen, von wo. Ein Audit-Log oder Loeschprotokoll erfuellt das.",
+ },
+
+ # ── L1: Ausnahmen / Sperren ───────────────────────────────────────
+ {
+ "id": "deletion_exceptions",
+ "label": "Ausnahmen + Sperrung statt Loeschung",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"(?:einschraenkung|einschränkung)\s+der\s+verarbeitung|art\.?\s*18",
+ r"sperr\w+\s+(?:statt|anstelle)\s+l(?:oe|ö)sch",
+ r"l(?:oe|ö)sch(?:beschr|sperr|ausnahme|hindernis)",
+ r"(?:rechtsstreit|gerichtsverfahren|prozessrelevant)",
+ ],
+ "severity": "MEDIUM",
+ "hint": "Wenn Loeschung nicht moeglich ist (laufender Prozess, gesetzliche Aufbewahrung, Streitfall) muss stattdessen Sperrung/Einschraenkung (Art. 18 DSGVO) erfolgen. Sperrkonzept dokumentieren.",
+ },
+
+ # ── L1: Review-Zyklus ─────────────────────────────────────────────
+ {
+ "id": "review_cycle",
+ "label": "Review-Zyklus / regelmaessige Pruefung",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"(?:jaehrlich|jährlich|halbjaehrlich|halbjährlich|vierteljaehrlich|vierteljährlich|quartalsweise)\s+(?:gepr|review|aktualis)",
+ r"review[\-\s]?(?:zyklus|intervall|frist)",
+ r"naechste|nächste\s+(?:ueberpr|überpr|review)",
+ r"(?:loeschkonzept|löschkonzept)\s+(?:wird|muss)\s+(?:regelmaessig|regelmäßig|jaehrlich|jährlich)",
+ ],
+ "severity": "LOW",
+ "hint": "DIN 66398 + Praxis: Loeschkonzept jaehrlich (oder bei Systemaenderungen ausserplanmaessig) ueberpruefen. Frist explizit benennen ('jaehrlich') statt nur 'regelmaessig'.",
+ },
+
+ # ── L1: DSGVO-Verweise ────────────────────────────────────────────
+ {
+ "id": "gdpr_reference",
+ "label": "Rechtliche Grundlagen referenziert",
+ "level": 1, "parent": None,
+ "patterns": [
+ r"art\.?\s*5\s*(?:abs\.?\s*1\s*)?(?:lit\.?\s*)?e\s*dsgvo",
+ r"art\.?\s*17\s*dsgvo",
+ r"art\.?\s*32\s*dsgvo",
+ r"speicherbegrenzung",
+ ],
+ "severity": "LOW",
+ "hint": "Direkte Norm-Referenzen erhoehen Beweiskraft: Art. 5(1)(e) DSGVO (Speicherbegrenzung), Art. 17 (Recht auf Loeschung), Art. 32 (TOMs).",
+ },
+]
diff --git a/backend-compliance/compliance/services/doc_checks/runner.py b/backend-compliance/compliance/services/doc_checks/runner.py
index a067f423..edc78b9f 100644
--- a/backend-compliance/compliance/services/doc_checks/runner.py
+++ b/backend-compliance/compliance/services/doc_checks/runner.py
@@ -20,6 +20,7 @@ from .avv_checks import AVV_CHECKLIST
from .scc_checks import SCC_CHECKLIST
from .tom_annex_checks import TOM_ANNEX_CHECKLIST
from .sub_processor_checks import SUB_PROCESSOR_LIST_CHECKLIST
+from .loeschkonzept_checks import LOESCHKONZEPT_CHECKLIST
logger = logging.getLogger(__name__)
@@ -52,6 +53,10 @@ _CHECKLIST_MAP = {
"sub_processor_list": (SUB_PROCESSOR_LIST_CHECKLIST, "Art. 28(3)(d) DSGVO"),
"sub_processor": (SUB_PROCESSOR_LIST_CHECKLIST, "Art. 28(3)(d) DSGVO"),
"unterauftragnehmer": (SUB_PROCESSOR_LIST_CHECKLIST, "Art. 28(3)(d) DSGVO"),
+ "loeschkonzept": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
+ "loeschung": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
+ "loeschfristen": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
+ "deletion_concept": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
}
diff --git a/backend-compliance/compliance/services/rag_document_checker.py b/backend-compliance/compliance/services/rag_document_checker.py
index a07a9d0c..6fc67524 100644
--- a/backend-compliance/compliance/services/rag_document_checker.py
+++ b/backend-compliance/compliance/services/rag_document_checker.py
@@ -241,8 +241,35 @@ def _map_doc_type(doc_type: str) -> str:
return _DOC_TYPE_MAP.get(doc_type, doc_type)
+# Doc-types that have no own MCs but can borrow from a related set.
+# (DB currently covers: dse, cookie, loeschkonzept, widerruf, dsfa,
+# avv, agb, impressum — total 1874 MCs across these.)
+_MC_ALIAS_FALLBACK = {
+ "nutzungsbedingungen": "agb", # T&C overlap
+ "terms": "agb",
+ "terms_of_use": "agb",
+ "social_media": "dse", # Joint-controller / Art. 26 is in DSE area
+ "joint_controller": "dse",
+ "sub_processor": "avv", # Subprocessor list = AVV annex
+ "sub_processor_list": "avv",
+ "scc": "avv", # SCC = AVV-Vertragsklauseln
+ "standardvertragsklauseln": "avv",
+ "tom_annex": "avv", # TOM-Annex meist als AVV-Anlage
+ "tom": "avv",
+ "dpa": "avv",
+ "loeschung": "loeschkonzept",
+ "loeschfristen": "loeschkonzept",
+ "eu_institution": "dse", # EU institution = DSE under VO 2018/1725
+ "dsb": "dse", # DSB info ist Teil der DSE
+}
+
+
async def _load_controls(doc_type: str, db_url: str, limit: int) -> list[dict]:
- """Load all doc_check_controls for a doc_type from PostgreSQL."""
+ """Load all doc_check_controls for a doc_type from PostgreSQL.
+
+ Falls back via _MC_ALIAS_FALLBACK when no MCs exist for the requested
+ type (e.g. 'nutzungsbedingungen' -> 'agb').
+ """
try:
import asyncpg
db = db_url or os.getenv(
@@ -264,6 +291,10 @@ async def _load_controls(doc_type: str, db_url: str, limit: int) -> list[dict]:
query += f" LIMIT {limit}"
rows = await conn.fetch(query, doc_type)
+ if not rows and doc_type in _MC_ALIAS_FALLBACK:
+ fallback = _MC_ALIAS_FALLBACK[doc_type]
+ logger.info("No MCs for %s -> falling back to %s", doc_type, fallback)
+ rows = await conn.fetch(query, fallback)
return [dict(r) for r in rows]
except Exception as e:
logger.warning("MC query failed: %s", e)
diff --git a/backend-compliance/scripts/backfill_mc_regulation.py b/backend-compliance/scripts/backfill_mc_regulation.py
new file mode 100644
index 00000000..bf730bc3
--- /dev/null
+++ b/backend-compliance/scripts/backfill_mc_regulation.py
@@ -0,0 +1,180 @@
+"""
+Backfill the doc_check_controls.regulation + .article fields.
+
+The fields are currently NULL on all 1874 rows. Many MCs cite the
+relevant norm inline in title / check_question / pass_criteria
+(e.g. 'Art. 6 Abs. 1 lit. a DSGVO', '§ 25 Abs. 1 TDDDG'). We detect
+those with regex and UPDATE the row.
+
+Run inside the bp-compliance-backend container:
+ docker exec bp-compliance-backend python3 /app/scripts/backfill_mc_regulation.py [--dry-run]
+
+The script is idempotent: existing non-null regulation is never overwritten.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+import sys
+
+import asyncpg
+
+
+# Ordered: first match wins. Each pattern captures (article_str, regulation_label).
+_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+ # Art. X DSGVO / GDPR / EU 2016/679
+ (re.compile(
+ r"\b(?:art\.?|artikel)\s*"
+ r"(\d+[a-z]?(?:\s*(?:abs\.?|absatz)\s*\d+)?"
+ r"(?:\s*(?:lit\.?|litera|buchstabe)\s*[a-z])?"
+ r"(?:\s*satz\s*\d+)?)"
+ r"\s*(?:dsgvo|gdpr|vo\s*\(eu\)\s*2016/679|eu[\s-]?vo\s*2016/679)",
+ re.I,
+ ), "DSGVO"),
+ # § X TDDDG / TTDSG
+ (re.compile(
+ r"§\s*(\d+[a-z]?(?:\s*abs\.?\s*\d+)?(?:\s*(?:nr\.?|lit\.?)\s*\w+)?)\s*"
+ r"(?:tdddg|ttdsg|tkg)",
+ re.I,
+ ), "TDDDG"),
+ # § X TMG
+ (re.compile(
+ r"§\s*(\d+[a-z]?(?:\s*abs\.?\s*\d+)?(?:\s*nr\.?\s*\d+)?)\s*tmg\b",
+ re.I,
+ ), "TMG"),
+ # § X BGB
+ (re.compile(
+ r"§\s*(\d+[a-z]?(?:\s*abs\.?\s*\d+)?)\s*bgb\b",
+ re.I,
+ ), "BGB"),
+ # § X HGB
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*hgb\b",
+ re.I,
+ ), "HGB"),
+ # § X AO
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*ao\b",
+ re.I,
+ ), "AO"),
+ # § X MStV (Medienstaatsvertrag)
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*m(?:edien)?st(?:aat)?v\b",
+ re.I,
+ ), "MStV"),
+ # § X UWG
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*uwg\b",
+ re.I,
+ ), "UWG"),
+ # § X VSBG (Verbraucherstreitbeilegung)
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*vsbg\b",
+ re.I,
+ ), "VSBG"),
+ # § X PAngV
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*p(?:reis)?ang?v\b",
+ re.I,
+ ), "PAngV"),
+ # § X GwG
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*gwg\b",
+ re.I,
+ ), "GwG"),
+ # § X BDSG
+ (re.compile(
+ r"§\s*(\d+[a-z]?)\s*bdsg\b",
+ re.I,
+ ), "BDSG"),
+ # EU-VO 524/2013 (ODR), 2018/1725 (EU-DSGVO) etc.
+ (re.compile(
+ r"\bart\.?\s*(\d+)\s*(?:eu[\s-]?vo|vo|verordnung)\s*(?:\(eu\)\s*)?(\d+/\d+)",
+ re.I,
+ ), "EU-VO"),
+ # Norm names without numbers, last resort (set article=NULL)
+ (re.compile(r"\bdsgvo\b", re.I), "DSGVO"),
+ (re.compile(r"\btdddg\b|\bttdsg\b", re.I), "TDDDG"),
+ (re.compile(r"\btmg\b", re.I), "TMG"),
+ (re.compile(r"\bbgb\b", re.I), "BGB"),
+ (re.compile(r"\bmstv\b", re.I), "MStV"),
+ (re.compile(r"\buwg\b", re.I), "UWG"),
+ (re.compile(r"\bvsbg\b", re.I), "VSBG"),
+ (re.compile(r"\bgwg\b", re.I), "GwG"),
+ (re.compile(r"\bbdsg\b", re.I), "BDSG"),
+]
+
+
+def detect(text: str) -> tuple[str | None, str | None]:
+ """Return (regulation, article) for the first pattern that matches."""
+ if not text:
+ return None, None
+ for pat, label in _PATTERNS:
+ m = pat.search(text)
+ if m:
+ article = m.group(1) if m.groups() else None
+ if label == "EU-VO" and m.lastindex and m.lastindex >= 2:
+ article = f"Art. {m.group(1)} EU-VO {m.group(2)}"
+ elif article:
+ article = re.sub(r"\s+", " ", article).strip()
+ return label, article
+ return None, None
+
+
+async def main(dry_run: bool = False) -> None:
+ db = os.getenv("DATABASE_URL")
+ if not db:
+ print("DATABASE_URL not set", file=sys.stderr)
+ sys.exit(1)
+ conn = await asyncpg.connect(db)
+
+ rows = await conn.fetch(
+ "SELECT id, title, check_question, pass_criteria::text AS pc "
+ "FROM compliance.doc_check_controls "
+ "WHERE regulation IS NULL"
+ )
+ print(f"{len(rows)} MCs with NULL regulation")
+
+ updates: list[tuple[str | None, str | None, str]] = []
+ hits = {"DSGVO": 0, "TDDDG": 0, "TMG": 0, "BGB": 0, "MStV": 0,
+ "UWG": 0, "VSBG": 0, "EU-VO": 0, "HGB": 0, "AO": 0,
+ "PAngV": 0, "GwG": 0, "BDSG": 0}
+ no_match = 0
+ for r in rows:
+ combined = " ".join(filter(None, [
+ r["title"] or "", r["check_question"] or "", r["pc"] or "",
+ ]))
+ reg, art = detect(combined)
+ if reg:
+ hits[reg] = hits.get(reg, 0) + 1
+ updates.append((reg, art, str(r["id"])))
+ else:
+ no_match += 1
+
+ print(f"Detected: {sum(hits.values())} | no match: {no_match}")
+ for k, v in sorted(hits.items(), key=lambda x: -x[1]):
+ if v:
+ print(f" {k:8s} {v:>5}")
+
+ if dry_run:
+ print("\nDRY RUN — no changes written. Re-run without --dry-run to apply.")
+ await conn.close()
+ return
+
+ # Apply updates in batches
+ BATCH = 200
+ for i in range(0, len(updates), BATCH):
+ chunk = updates[i:i + BATCH]
+ await conn.executemany(
+ "UPDATE compliance.doc_check_controls "
+ "SET regulation = $1, article = $2 WHERE id = $3::uuid",
+ chunk,
+ )
+ print(f"\nApplied {len(updates)} updates.")
+ await conn.close()
+
+
+if __name__ == "__main__":
+ asyncio.run(main(dry_run="--dry-run" in sys.argv))