Benjamin Admin
6c5e086356
Dedup fixes: - Anchor links (#cookies, #betroffenenrechte) on same page are skipped entirely - Noise titles filtered: 'drucken', 'nach oben', 'Datenschutz' (too generic) - Documents with < 50 words filtered (navigation snippets) - Documents with identical word_count merged (same page, different title) - URL-only titles filtered False positive fixes (dsi_document_checker.py): - 'Kontaktdaten des Verantwortlichen' pattern for controller check - 'Zweck und Rechtsgrundlage' combined heading pattern - 'Welche Daten werden verarbeitet' question-style headings - 'Betroffenenrechte' as standalone heading - 'Welche Rechte hat der Betroffene' question pattern - 'Daten werden geloescht' retention pattern - 'Auftragsverarbeiter' as recipient indicator Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
246 lines
9.1 KiB
Python
246 lines
9.1 KiB
Python
"""
|
|
DSI Document Checker — validates discovered legal documents against
|
|
mandatory content requirements.
|
|
|
|
Checks each document type against its specific legal requirements:
|
|
- Datenschutzinformation: Art. 13/14 DSGVO (9 Pflichtangaben)
|
|
- AGB: §305ff BGB
|
|
- Widerrufsbelehrung: §355, §312g BGB
|
|
- Cookie-Richtlinie: §25 TDDDG
|
|
- Impressum: §5 TMG / §18 MStV
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Art. 13 DSGVO mandatory fields for privacy policies
|
|
ART13_CHECKLIST = [
|
|
{
|
|
"id": "controller",
|
|
"label": "Verantwortlicher (Art. 13(1)(a))",
|
|
"patterns": [
|
|
r"verantwortlich\w*\s+(?:ist|im sinne|fuer|f(?:ue|ü)r)",
|
|
r"kontaktdaten\s+des\s+verantwortlichen",
|
|
r"name\s+(?:und|&)\s+kontaktdaten\s+des",
|
|
r"controller", r"verantwortliche\s+stelle",
|
|
r"responsible\s+(?:party|for)",
|
|
r"ihk\s+\w+\s+bodensee", # IHK-specific: org name as controller
|
|
],
|
|
"severity": "HIGH",
|
|
},
|
|
{
|
|
"id": "dpo",
|
|
"label": "Datenschutzbeauftragter (Art. 13(1)(b))",
|
|
"patterns": [
|
|
r"datenschutzbeauftragt", r"data\s+protection\s+officer",
|
|
r"kontaktdaten\s+de[rs]\s+(?:behördlichen\s+)?datenschutz",
|
|
r"dsb", r"dpo",
|
|
],
|
|
"severity": "MEDIUM",
|
|
},
|
|
{
|
|
"id": "purposes",
|
|
"label": "Zwecke der Verarbeitung (Art. 13(1)(c))",
|
|
"patterns": [
|
|
r"zweck\w*\s+(?:der|und|die)\s+(?:verarbeitung|datenerhebung|datenverarbeitung|rechtsgrundlage)",
|
|
r"purpose\w*\s+(?:of|for)\s+(?:processing|data)",
|
|
r"zu\s+welch\w+\s+zweck",
|
|
r"welche\s+daten\s+werden.*verarbeitet",
|
|
r"daten\s+werden\s+(?:zu|fuer|für)\s+(?:folgende|diese)",
|
|
],
|
|
"severity": "HIGH",
|
|
},
|
|
{
|
|
"id": "legal_basis",
|
|
"label": "Rechtsgrundlage (Art. 13(1)(c))",
|
|
"patterns": [
|
|
r"rechtsgrundlage", r"art\.\s*6\s*(?:abs|absatz)?\s*\.?\s*1",
|
|
r"legal\s+basis", r"berechtigtes\s+interesse",
|
|
r"auf\s+grundlage\s+(?:von|des|der)\s+(?:art|§)",
|
|
r"lit\.\s*[a-f]\)",
|
|
],
|
|
"severity": "HIGH",
|
|
},
|
|
{
|
|
"id": "recipients",
|
|
"label": "Empfaenger (Art. 13(1)(e))",
|
|
"patterns": [
|
|
r"empf(?:ae|ä)nger", r"(?:ueber|über|weiter)mitt(?:el|l)ung",
|
|
r"recipient", r"weitergabe\s+(?:an|von)\s+daten",
|
|
r"dritte", r"third\s+part",
|
|
r"welche\s+daten\s+werden\s+(?:ueber|über)mittelt",
|
|
r"auftragsverarbeit",
|
|
],
|
|
"severity": "MEDIUM",
|
|
},
|
|
{
|
|
"id": "third_country",
|
|
"label": "Drittlandtransfer (Art. 13(1)(f))",
|
|
"patterns": [
|
|
r"drittland", r"dritt\s*staat", r"drittl(?:ae|ä)nder",
|
|
r"third\s+countr", r"angemessenheitsbeschluss",
|
|
r"standard\s*vertragsklausel", r"scc",
|
|
],
|
|
"severity": "MEDIUM",
|
|
},
|
|
{
|
|
"id": "retention",
|
|
"label": "Speicherdauer (Art. 13(2)(a))",
|
|
"patterns": [
|
|
r"speicherdauer", r"aufbewahrungsfrist",
|
|
r"(?:wie\s+lange|dauer)\s+(?:werden|gespeichert)",
|
|
r"retention\s+period", r"l(?:oe|ö)sch(?:ung|frist|konzept)",
|
|
r"wie\s+lange\s+werden\s+die\s+daten\s+aufbewahrt",
|
|
r"daten\s+werden\s+gel(?:oe|ö)scht",
|
|
r"(?:\d+\s+(?:tage|monate|jahre)|nach\s+\d+)",
|
|
],
|
|
"severity": "HIGH",
|
|
},
|
|
{
|
|
"id": "rights",
|
|
"label": "Betroffenenrechte (Art. 13(2)(b))",
|
|
"patterns": [
|
|
r"recht\s+auf\s+auskunft", r"recht\s+auf\s+l(?:oe|ö)schung",
|
|
r"recht\s+auf\s+berichtigung", r"widerspruchsrecht",
|
|
r"art\.\s*1[5-9]", r"art\.\s*2[0-2]",
|
|
r"right\s+to\s+(?:access|erasure|rectification|object)",
|
|
r"betroffenenrecht", r"rechte\s+(?:des|der)\s+betroffenen",
|
|
r"welche\s+rechte\s+ha(?:t|ben)\s+(?:der|die|sie)",
|
|
r"ihnen\s+(?:stehen|steht)\s+(?:ein|folgende)\s+recht",
|
|
],
|
|
"severity": "HIGH",
|
|
},
|
|
{
|
|
"id": "complaint",
|
|
"label": "Beschwerderecht (Art. 13(2)(d))",
|
|
"patterns": [
|
|
r"beschwerderecht", r"aufsichtsbeh(?:oe|ö)rde",
|
|
r"right\s+to\s+lodge\s+a\s+complaint",
|
|
r"supervisory\s+authority", r"datenschutzbeh(?:oe|ö)rde",
|
|
],
|
|
"severity": "MEDIUM",
|
|
},
|
|
]
|
|
|
|
# §355 BGB requirements for cancellation/withdrawal policies
|
|
WIDERRUF_CHECKLIST = [
|
|
{"id": "right_info", "label": "Belehrung ueber Widerrufsrecht",
|
|
"patterns": [r"widerrufsrecht", r"right\s+of\s+withdrawal", r"recht\s+(?:zum|auf)\s+widerruf"]},
|
|
{"id": "deadline", "label": "Widerrufsfrist (14 Tage)",
|
|
"patterns": [r"14\s+tage", r"vierzehn\s+tage", r"14\s+days", r"fourteen\s+days"]},
|
|
{"id": "form", "label": "Form des Widerrufs",
|
|
"patterns": [r"widerrufsformular", r"muster.?widerruf", r"withdrawal\s+form", r"formular"]},
|
|
{"id": "consequences", "label": "Folgen des Widerrufs",
|
|
"patterns": [r"folgen\s+des\s+widerrufs", r"consequences\s+of\s+withdrawal", r"rueckerstattung"]},
|
|
]
|
|
|
|
# AGB minimal requirements
|
|
AGB_CHECKLIST = [
|
|
{"id": "scope", "label": "Geltungsbereich",
|
|
"patterns": [r"geltungsbereich", r"geltung", r"scope", r"diese\s+(?:agb|bedingungen)\s+gelten"]},
|
|
{"id": "contract", "label": "Vertragsschluss",
|
|
"patterns": [r"vertragsschluss", r"zustandekommen", r"contract\s+formation", r"angebot\s+und\s+annahme"]},
|
|
{"id": "liability", "label": "Haftung",
|
|
"patterns": [r"haftung", r"liability", r"schadensersatz", r"haftungsbeschr(?:ae|ä)nkung"]},
|
|
{"id": "jurisdiction", "label": "Gerichtsstand / Anwendbares Recht",
|
|
"patterns": [r"gerichtsstand", r"anwendbares\s+recht", r"jurisdiction", r"governing\s+law"]},
|
|
]
|
|
|
|
|
|
def check_document_completeness(
|
|
text: str,
|
|
doc_type: str,
|
|
doc_title: str,
|
|
doc_url: str,
|
|
) -> list[dict]:
|
|
"""Check a legal document against its type-specific requirements.
|
|
|
|
Returns a list of findings (missing/present fields).
|
|
"""
|
|
findings = []
|
|
text_lower = text.lower()
|
|
|
|
if not text or len(text) < 50:
|
|
findings.append({
|
|
"code": f"DSI-EMPTY-{doc_type.upper()}",
|
|
"severity": "HIGH",
|
|
"text": f"Dokument '{doc_title}' ist leer oder zu kurz fuer eine Pruefung.",
|
|
"doc_title": doc_title,
|
|
"doc_url": doc_url,
|
|
"doc_type": doc_type,
|
|
})
|
|
return findings
|
|
|
|
# Select checklist based on document type
|
|
if doc_type in ("dse", "datenschutz", "privacy"):
|
|
checklist = ART13_CHECKLIST
|
|
label = "Art. 13 DSGVO"
|
|
elif doc_type in ("widerruf", "withdrawal", "cancellation"):
|
|
checklist = WIDERRUF_CHECKLIST
|
|
label = "§355 BGB"
|
|
elif doc_type in ("agb", "terms", "nutzungsbedingungen"):
|
|
checklist = AGB_CHECKLIST
|
|
label = "§305ff BGB"
|
|
else:
|
|
checklist = ART13_CHECKLIST # Default: check as DSE
|
|
label = "Art. 13 DSGVO"
|
|
|
|
present = 0
|
|
total = len(checklist)
|
|
for check in checklist:
|
|
found = any(re.search(p, text_lower) for p in check["patterns"])
|
|
if not found:
|
|
findings.append({
|
|
"code": f"DSI-MISSING-{check['id'].upper()}",
|
|
"severity": check.get("severity", "MEDIUM"),
|
|
"text": (
|
|
f"'{doc_title}': Pflichtangabe '{check['label']}' nicht gefunden. "
|
|
f"Erforderlich nach {label}."
|
|
),
|
|
"doc_title": doc_title,
|
|
"doc_url": doc_url,
|
|
"doc_type": doc_type,
|
|
"check_id": check["id"],
|
|
})
|
|
else:
|
|
present += 1
|
|
|
|
# Add summary finding
|
|
if total > 0:
|
|
pct = round(present / total * 100)
|
|
if pct < 100:
|
|
findings.insert(0, {
|
|
"code": f"DSI-SCORE-{doc_type.upper()}",
|
|
"severity": "LOW" if pct >= 80 else "MEDIUM" if pct >= 50 else "HIGH",
|
|
"text": (
|
|
f"'{doc_title}': {present}/{total} Pflichtangaben vorhanden ({pct}%). "
|
|
f"Fehlend: {total - present} Angaben nach {label}."
|
|
),
|
|
"doc_title": doc_title,
|
|
"doc_url": doc_url,
|
|
"doc_type": doc_type,
|
|
})
|
|
|
|
return findings
|
|
|
|
|
|
def classify_document_type(title: str, url: str) -> str:
|
|
"""Classify a document by its title/URL into a legal document type."""
|
|
combined = f"{title} {url}".lower()
|
|
|
|
if any(kw in combined for kw in ["datenschutz", "privacy", "dsgvo", "data protection", "données"]):
|
|
return "dse"
|
|
if any(kw in combined for kw in ["widerruf", "withdrawal", "rétractation", "desistimiento"]):
|
|
return "widerruf"
|
|
if any(kw in combined for kw in ["agb", "allgemeine geschäftsbedingungen", "terms",
|
|
"nutzungsbedingungen", "conditions"]):
|
|
return "agb"
|
|
if any(kw in combined for kw in ["cookie", "slapuk", "evästeet", "kakor"]):
|
|
return "cookie"
|
|
if any(kw in combined for kw in ["impressum", "imprint", "legal notice", "mentions légales"]):
|
|
return "impressum"
|
|
return "other"
|