feat(cookie-check): enhanced patterns + active opt-out link validator
cookie_checks.py: - cookie_names_listed: now also matches CMP placeholder notation (BMW: 'Adfpc###', 'CT###') and 'Diese Datenverarbeitung verwendet die folgenden Cookies oder ähnliche Technologien' as list-shape signal. Cryptic vendor names like 'audience', 'adformfrpid' are accepted via the surrounding markup, not by hard-coding each one. - cookie_providers_named: new pattern 'Gesetzt von: <Firma>' (BMW/ePaaS per-cookie vendor naming) + recognition of full legal-form names (Adform A/S, BMW AG, Adobe Systems Software Ireland Limited). - cookie_duration_values: now matches 'Ablauf: 1 Jahr' / 'Speicherdauer: 30 Tage' (BMW format) in addition to the legacy '<n> <unit>'. New L1 + L2 checks for controller in cookie-policy: - cookie_controller (L1): the cookie policy must name Verantwortlich(er) - cookie_controller_address (L2): PLZ + Ort or address keywords - cookie_controller_contact_or_link (L2): email/phone OR link back to Datenschutzerklärung (the practical equivalent — BMW does this) New L2 checks (parented under opt_out): - cookie_optout_links: detects per-provider opt-out URLs in the text - cookie_privacy_policy_links: per-provider privacy-policy URLs New service: cookie_link_validator.py - extract_links(text): pulls all https?://… URLs that follow 'Opt-Out Link:' / 'Link zur Privacy Policy:' (deduped) - validate_links(links): probes every URL concurrently (HEAD first, GET fallback for 405/403). 10 parallel, 8s per request, 60s batch cap. Returns reachable=True/False + status + final_url. - build_check_items(): renders 2 CheckItems (opt-out + privacy-policy), each pass if ALL links 2xx/3xx, fail with up-to-5 broken-link examples. Hook in _check_single: doc_type=='cookie' triggers the validator after regex+MC checks. Recomputes correctness with the new L2 items. This addresses two concrete BMW observations: 1. BMW's per-cookie structure (Name + Zweck + Ablauf, Gesetzt von: …, Opt-Out Link: …) now recognised → 'Konkrete Cookie-Namen aufgelistet' and 'Konkrete Speicherdauern' should pass. 2. Defective opt-out URLs surface as compliance findings rather than silently passing — Art. 7(3) DSGVO requires a working withdrawal path per provider.
This commit is contained in:
Benjamin Admin
@@ -701,6 +701,29 @@ async def _check_single(
|
||||
except Exception as e:
|
||||
logger.warning("LLM verification skipped: %s", e)
|
||||
|
||||
# Cookie-policy only: actively HTTP-probe the Opt-Out + Privacy-Policy
|
||||
# URLs the document advertises. Broken links make individual provider
|
||||
# entries non-compliant under Art. 7(3) DSGVO.
|
||||
if doc_type == "cookie":
|
||||
try:
|
||||
from compliance.services.cookie_link_validator import (
|
||||
extract_links, validate_links, build_check_items,
|
||||
)
|
||||
links = extract_links(text)
|
||||
if links:
|
||||
logger.info("Cookie-link validator: %d urls extracted from %s",
|
||||
len(links), label)
|
||||
validated = await validate_links(links)
|
||||
for item in build_check_items(validated):
|
||||
all_checks.append(CheckItem(**item))
|
||||
# Re-compute correctness with the new L2 items
|
||||
l2_active = [c for c in all_checks if c.level == 2 and not c.skipped]
|
||||
l2_passed = sum(1 for c in l2_active if c.passed)
|
||||
if l2_active:
|
||||
correctness = round(l2_passed / len(l2_active) * 100)
|
||||
except Exception as e:
|
||||
logger.warning("Cookie-link validation skipped for %s: %s", label, e)
|
||||
|
||||
non_score = [f for f in findings if "SCORE" not in f.get("code", "")]
|
||||
return DocCheckResult(
|
||||
label=label, url=url, doc_type=doc_type,
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
"""
|
||||
Cookie-Richtlinie Opt-Out and Privacy-Policy link validator.
|
||||
|
||||
Art. 7(3) DSGVO: "Der Widerruf der Einwilligung muss so einfach wie die
|
||||
Erteilung sein". Per third-party provider in the cookie policy there must
|
||||
be a working opt-out mechanism. A missing or broken link makes that
|
||||
provider entry legally non-compliant.
|
||||
|
||||
This module extracts the URLs from the cookie-policy text and tests each
|
||||
one via async HTTP (HEAD first, GET fallback). Returns structured findings
|
||||
the route layer turns into CheckItems for the email + frontend report.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from typing import TypedDict
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# URL extraction patterns. Each captures the URL that follows the keyword.
|
||||
_URL_RE = r"https?://[\w\-./?#&=:%~+@]+"
|
||||
_OPTOUT_PATTERN = re.compile(
|
||||
rf"opt[\-\s]?out[\-\s]?(?:link)?\s*[:\|]?\s*({_URL_RE})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_PRIVACY_PATTERN = re.compile(
|
||||
rf"(?:link\s+zur?\s+(?:privacy[\-\s]?policy|datenschutz\w*)|privacy[\-\s]?policy)\s*[:\|]?\s*({_URL_RE})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Concurrency + timeout budget. 10 parallel requests, 8s per request,
|
||||
# whole batch capped at 60s — keeps the cookie check inside the existing
|
||||
# 120s backend → consent-tester budget.
|
||||
_MAX_CONCURRENT = 10
|
||||
_PER_URL_TIMEOUT = 8.0
|
||||
_BATCH_TIMEOUT = 60.0
|
||||
|
||||
|
||||
class LinkCheck(TypedDict, total=False):
|
||||
url: str
|
||||
kind: str # "opt-out" | "privacy-policy"
|
||||
status: int # 0 = unreachable
|
||||
final_url: str
|
||||
error: str
|
||||
reachable: bool
|
||||
|
||||
|
||||
def extract_links(text: str) -> list[LinkCheck]:
|
||||
"""Pull all Opt-Out + Privacy-Policy URLs from a cookie-policy text.
|
||||
|
||||
Deduplicates by URL+kind. Strips trailing punctuation/quotes commonly
|
||||
captured by greedy URL regex.
|
||||
"""
|
||||
found: dict[tuple[str, str], LinkCheck] = {}
|
||||
for kind, pattern in (("opt-out", _OPTOUT_PATTERN),
|
||||
("privacy-policy", _PRIVACY_PATTERN)):
|
||||
for match in pattern.finditer(text):
|
||||
url = match.group(1).rstrip(".,;:\"')(]").strip()
|
||||
if not url.startswith(("http://", "https://")):
|
||||
continue
|
||||
key = (url, kind)
|
||||
if key not in found:
|
||||
found[key] = LinkCheck(url=url, kind=kind)
|
||||
return list(found.values())
|
||||
|
||||
|
||||
async def validate_links(links: list[LinkCheck]) -> list[LinkCheck]:
|
||||
"""HTTP-probe each link concurrently. Adds status + reachable flag.
|
||||
|
||||
Uses HEAD first (fast), falls back to GET for servers that reject HEAD.
|
||||
Accepts any 2xx/3xx as reachable; 4xx/5xx and timeouts as broken.
|
||||
"""
|
||||
if not links:
|
||||
return []
|
||||
sem = asyncio.Semaphore(_MAX_CONCURRENT)
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=_PER_URL_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": "BreakPilot-LinkChecker/1.0"},
|
||||
) as client:
|
||||
async def probe(link: LinkCheck) -> LinkCheck:
|
||||
async with sem:
|
||||
try:
|
||||
resp = await client.head(link["url"])
|
||||
if resp.status_code in (405, 403):
|
||||
# Some servers reject HEAD; try GET
|
||||
resp = await client.get(link["url"])
|
||||
link["status"] = resp.status_code
|
||||
link["final_url"] = str(resp.url)
|
||||
link["reachable"] = 200 <= resp.status_code < 400
|
||||
except httpx.TimeoutException:
|
||||
link["status"] = 0
|
||||
link["error"] = "timeout"
|
||||
link["reachable"] = False
|
||||
except Exception as e:
|
||||
link["status"] = 0
|
||||
link["error"] = str(e)[:80]
|
||||
link["reachable"] = False
|
||||
return link
|
||||
|
||||
try:
|
||||
results = await asyncio.wait_for(
|
||||
asyncio.gather(*[probe(link) for link in links]),
|
||||
timeout=_BATCH_TIMEOUT,
|
||||
)
|
||||
return list(results)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"Cookie-link batch timeout after %.0fs — %d urls",
|
||||
_BATCH_TIMEOUT, len(links),
|
||||
)
|
||||
# Best-effort: return whatever links got updated
|
||||
return links
|
||||
|
||||
|
||||
# ── CheckItem rendering ──────────────────────────────────────────────
|
||||
|
||||
def build_check_items(validated: list[LinkCheck]) -> list[dict]:
|
||||
"""Turn validator results into compliance-check items (one per kind).
|
||||
|
||||
Always returns 2 items (opt-out + privacy-policy) so the report layout
|
||||
is stable. Skipped if no links of that kind were extracted.
|
||||
"""
|
||||
items: list[dict] = []
|
||||
for kind, label in (
|
||||
("opt-out", "Opt-Out-Links der Drittanbieter erreichbar"),
|
||||
("privacy-policy", "Privacy-Policy-Links der Drittanbieter erreichbar"),
|
||||
):
|
||||
of_kind = [l for l in validated if l.get("kind") == kind]
|
||||
if not of_kind:
|
||||
continue
|
||||
total = len(of_kind)
|
||||
ok = sum(1 for l in of_kind if l.get("reachable"))
|
||||
broken = [l for l in of_kind if not l.get("reachable")]
|
||||
all_pass = ok == total
|
||||
|
||||
hint = ""
|
||||
matched = ""
|
||||
if all_pass:
|
||||
matched = f"{ok}/{total} Links erreichbar (HTTP 2xx/3xx)"
|
||||
else:
|
||||
broken_summary = ", ".join(
|
||||
f"{l['url'][:60]} ({l.get('status') or l.get('error', '?')})"
|
||||
for l in broken[:5]
|
||||
)
|
||||
hint = (
|
||||
f"{len(broken)}/{total} Links sind defekt. Defekte "
|
||||
f"Provider-Eintraege erfuellen Art. 7(3) DSGVO nicht — der "
|
||||
f"Widerruf der Einwilligung ist fuer diese Anbieter unmoeglich. "
|
||||
f"Beispiele: {broken_summary}"
|
||||
)
|
||||
items.append({
|
||||
"id": f"cookie_links_{kind.replace('-', '_')}",
|
||||
"label": label,
|
||||
"passed": all_pass,
|
||||
"severity": "MEDIUM" if kind == "opt-out" else "LOW",
|
||||
"matched_text": matched,
|
||||
"level": 2,
|
||||
"parent": "opt_out",
|
||||
"skipped": False,
|
||||
"hint": hint,
|
||||
})
|
||||
return items
|
||||
@@ -23,13 +23,21 @@ COOKIE_CHECKLIST = [
|
||||
"label": "Konkrete Cookie-Namen aufgelistet",
|
||||
"level": 2, "parent": "cookie_types",
|
||||
"patterns": [
|
||||
# Standard cookie names
|
||||
r"(?:_ga|_gid|_gat|_fbp|_gcl|phpsessid|php\s+session\s+id|jsessionid|csrf|xsrf|cookieinfo|et_id|bt_\w+|cntcookie|shophk)",
|
||||
# CMP-placeholder notation (e.g. BMW: Adfpc###, CT###, EBFC###)
|
||||
r"\b[a-z][a-z0-9_\-]*#{2,}\b",
|
||||
# "Diese Datenverarbeitung verwendet die folgenden Cookies..."
|
||||
# (BMW/ePaaS pattern — strong signal that a list follows)
|
||||
r"folgende(?:n)?\s+cookies?\s+oder\s+(?:aehnliche|ähnliche)\s+technologien",
|
||||
r"diese\s+datenverarbeitung\s+verwendet\s+(?:die\s+)?folgende(?:n)?\s+cookies?",
|
||||
# Header-row signals (Name | Zweck | Ablauf / "Name des Cookie")
|
||||
r"cookie[\-_]?name\s*[:\|]",
|
||||
r"name\s+des\s+cookie",
|
||||
r"(?:name|bezeichnung)\s+.*(?:funktion|zweck|speicherdauer|laufzeit)",
|
||||
],
|
||||
"severity": "MEDIUM",
|
||||
"hint": "Die DSK fordert eine Auflistung jedes einzelnen Cookies mit technischem Namen (z.B. _ga, _gid, PHPSESSID). Tipp: Browser-DevTools > Application > Cookies zeigt alle aktiven Cookies — gleichen Sie diese mit Ihrer Liste ab.",
|
||||
"hint": "Die DSK fordert eine Auflistung jedes einzelnen Cookies mit technischem Namen (z.B. _ga, _gid, PHPSESSID). Cryptic Vendor-IDs (audience, adformfrpid) oder Platzhalter-Notation (Adfpc###) sind ebenfalls zulaessig, solange klar ist welches Cookie gemeint ist. Browser-DevTools > Application > Cookies zeigt die echten Namen — gleichen Sie diese mit Ihrer Liste ab.",
|
||||
},
|
||||
{
|
||||
"id": "cookie_essential_justified",
|
||||
@@ -69,9 +77,13 @@ COOKIE_CHECKLIST = [
|
||||
"patterns": [
|
||||
r"(?:google\s+(?:analytics|tag\s+manager|ads)|matomo|piwik|hotjar|hubspot|facebook\s+pixel|meta\s+pixel|linkedin\s+insight|microsoft\s+clarity)",
|
||||
r"(?:anbieter|provider|dienst)\s*[:\|]\s*[A-Z]",
|
||||
# BMW/ePaaS pattern: "Gesetzt von: <Firma>" — per-cookie vendor naming
|
||||
r"gesetzt\s+von\s*[:\|]\s*[A-Z]",
|
||||
# Vendor full names with legal form ("Adform A/S", "BMW AG", ...)
|
||||
r"\b[A-Z][a-zA-ZÀ-ž]+(?:\s+[A-Z][a-zA-ZÀ-ž]+)*\s+(?:GmbH|AG|A/S|S\.A\.|Inc\.|Ltd\.|LLC|B\.V\.|Limited|Corp\.)\b",
|
||||
],
|
||||
"severity": "MEDIUM",
|
||||
"hint": "Art. 13 Abs. 1 lit. e DSGVO verlangt die Nennung der Empfaenger. Jeder Dienst, der Cookies setzt, muss mit Firmenname und Sitz benannt werden (z.B. 'Google Ireland Limited, Dublin'). Anonyme Angaben wie 'Drittanbieter' genuegen nicht.",
|
||||
"hint": "Art. 13 Abs. 1 lit. e DSGVO verlangt die Nennung der Empfaenger. Jeder Dienst, der Cookies setzt, muss mit Firmenname und Sitz benannt werden (z.B. 'Google Ireland Limited, Dublin' oder 'Gesetzt von: Adobe Systems Software Ireland Limited'). Anonyme Angaben wie 'Drittanbieter' genuegen nicht.",
|
||||
},
|
||||
{
|
||||
"id": "cookie_analytics_named",
|
||||
@@ -113,10 +125,11 @@ COOKIE_CHECKLIST = [
|
||||
"patterns": [
|
||||
r"\d+\s+(?:tag|monat|jahr|minute|stunde|day|month|year)",
|
||||
r"session[\-\s]?cookie",
|
||||
r"(?:ablauf|expiry|laufzeit)\s*[:\|]\s*\d+",
|
||||
# BMW/ePaaS pattern: "Ablauf: 1 Jahr", "Ablauf: 30 Tage", "Speicherdauer: 60 Tage"
|
||||
r"(?:ablauf|expiry|laufzeit|speicherdauer)\s*[:\|\s]+\d+\s*(?:tag|monat|jahr|minute|stunde|day|month|year)",
|
||||
],
|
||||
"severity": "LOW",
|
||||
"hint": "Die DSK-Orientierungshilfe verlangt die Speicherdauer pro Cookie (z.B. '_ga: 2 Jahre', '_gid: 24 Stunden'). Pruefen Sie die tatsaechlichen Werte in den Browser-DevTools — Anbieter-Dokumentation ist oft veraltet.",
|
||||
"hint": "Die DSK-Orientierungshilfe verlangt die Speicherdauer pro Cookie (z.B. '_ga: 2 Jahre', '_gid: 24 Stunden' oder 'Ablauf: 1 Jahr'). Pruefen Sie die tatsaechlichen Werte in den Browser-DevTools — Anbieter-Dokumentation ist oft veraltet.",
|
||||
},
|
||||
|
||||
# ── L1: Drittanbieter ─────────────────────────────────────────────
|
||||
@@ -184,6 +197,76 @@ COOKIE_CHECKLIST = [
|
||||
"hint": "Ergaenzen Sie einen Hinweis auf Browser-Einstellungen mit konkreten Links zu den Hilfeseiten (Chrome, Firefox, Safari, Edge). Achtung: Browser-Einstellungen allein ersetzen nicht das Consent-Banner — §25 TDDDG verlangt aktive Einwilligung.",
|
||||
},
|
||||
|
||||
# ── L1: Verantwortlicher in der Cookie-Richtlinie ────────────────
|
||||
{
|
||||
"id": "cookie_controller",
|
||||
"label": "Verantwortlicher (Art. 13(1)(a) DSGVO)",
|
||||
"level": 1, "parent": None,
|
||||
"patterns": [
|
||||
r"verantwortlich\w*\s+(?:im\s+)?(?:datenschutzrechtlich|datenschutzrelevant|sinne?|sinn)",
|
||||
r"(?:datenschutz[\-]?rechtlich(?:er)?\s+)?verantwortlich\w*\s*[:\|]",
|
||||
r"daten(?:schutz)?[\-]?(?:rechtlich(?:er)?\s+)?(?:verantwortl|controller)",
|
||||
r"\bcontroller\b.*\b(?:art\.?\s*13|art\.?\s*14|gdpr|dsgvo)",
|
||||
],
|
||||
"severity": "MEDIUM",
|
||||
"hint": "Art. 13(1)(a) DSGVO verlangt die Nennung des Verantwortlichen in der Cookie-Richtlinie. Pflicht: Firmenname + Anschrift + Kontaktdaten (E-Mail/Telefon). Akzeptabel: knapper Verweis 'Details zum Verantwortlichen siehe Datenschutzerklaerung [Link]' wenn die DSI verlinkt ist.",
|
||||
},
|
||||
{
|
||||
"id": "cookie_controller_address",
|
||||
"label": "Anschrift des Verantwortlichen (PLZ + Ort)",
|
||||
"level": 2, "parent": "cookie_controller",
|
||||
"patterns": [
|
||||
# German postal code + city (5 digits + word)
|
||||
r"\b\d{5}\s+[A-Z][a-zA-ZÀ-ž\-]+",
|
||||
# Address keywords near PLZ
|
||||
r"(?:anschrift|adresse|sitz|hauptsitz|petuelring|strasse|str\.|platz)",
|
||||
],
|
||||
"severity": "LOW",
|
||||
"hint": "Die Adresse des Verantwortlichen (Strasse + PLZ + Ort) sollte in der Cookie-Richtlinie genannt werden, oder es muss ein klarer Link zur Datenschutzerklaerung mit vollstaendigen Kontaktdaten vorhanden sein. Nur 'Firma XY' ohne Anschrift erfuellt Art. 13(1)(a) DSGVO nicht.",
|
||||
},
|
||||
{
|
||||
"id": "cookie_controller_contact_or_link",
|
||||
"label": "Kontakt (E-Mail/Telefon) oder Verweis zur Datenschutzerklaerung",
|
||||
"level": 2, "parent": "cookie_controller",
|
||||
"patterns": [
|
||||
# E-mail
|
||||
r"\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b",
|
||||
# Phone (German format)
|
||||
r"(?:tel\.?|telefon|fon)[:\s\.]*(?:\+?\d[\d\s\-\/\(\)]{7,})",
|
||||
# Link to Datenschutzerklaerung (within cookie policy)
|
||||
r"(?:datenschutzerkl(?:ae|ä)rung|datenschutzhinweis|datenschutzinformation|privacy.?policy|privacy.?notice).*(?:href|http|www\.|/de/)",
|
||||
r"(?:siehe|details|weitere\s+informationen).*datenschutz",
|
||||
],
|
||||
"severity": "LOW",
|
||||
"hint": "Pflicht aus Art. 13(1)(b) DSGVO: entweder E-Mail/Telefon des Verantwortlichen direkt in der Cookie-Richtlinie ODER expliziter Hinweis 'Details in der Datenschutzerklaerung' mit klickbarem Link.",
|
||||
},
|
||||
|
||||
# ── L1: Opt-Out-Mechanismen pro Anbieter ──────────────────────────
|
||||
{
|
||||
"id": "cookie_optout_links",
|
||||
"label": "Opt-Out-Links pro Drittanbieter (Art. 7(3) DSGVO)",
|
||||
"level": 2, "parent": "opt_out",
|
||||
"patterns": [
|
||||
r"opt[\-\s]?out[\-\s]?(?:link|url)",
|
||||
r"(?:opt[\-\s]?out|widerruf|abmelden).*https?://",
|
||||
r"https?://[^\s]+(?:opt[\-\s]?out|optout|abmeld|widerruf)",
|
||||
],
|
||||
"severity": "MEDIUM",
|
||||
"hint": "Art. 7(3) DSGVO: Der Widerruf der Einwilligung muss so einfach wie die Erteilung sein. Pro Drittanbieter (Adobe, Adform, Meta, etc.) muss ein direkter Opt-Out-Link angegeben sein. Tipp: Konsolidiert ueber das Consent-Banner ('Einstellungen aendern') ist optimal; einzelne Provider-Opt-Out-Links sind als Backup wichtig.",
|
||||
},
|
||||
{
|
||||
"id": "cookie_privacy_policy_links",
|
||||
"label": "Privacy-Policy-Links pro Drittanbieter",
|
||||
"level": 2, "parent": "opt_out",
|
||||
"patterns": [
|
||||
r"link\s+zur?\s+(?:privacy|datenschutz)",
|
||||
r"datenschutzerkl(?:ae|ä)rung\s+(?:von|des|der)\s+\w+",
|
||||
r"(?:privacy\s+policy|privacy\s+notice)\s*[:\|]?\s*https?://",
|
||||
],
|
||||
"severity": "LOW",
|
||||
"hint": "Pro Drittanbieter sollte ein Link zur jeweiligen Datenschutzerklaerung des Anbieters gesetzt sein. Erlaubt dem Nutzer, die Datenverarbeitung beim Drittanbieter eigenverantwortlich nachzuvollziehen — Art. 13(2)(f) DSGVO Transparenzgebot.",
|
||||
},
|
||||
|
||||
# ── Neue L1: Cookie-Tabelle ───────────────────────────────────────
|
||||
{
|
||||
"id": "cookie_table",
|
||||
|
||||
Reference in New Issue
Block a user