Merge feat/zeroclaw-compliance-agent into main

Brings all compliance doc-check features: - 162 regex checks + 1874 Master Controls - LLM-agnostic agent with tool calling - Banner check (46 checks, 30 CMPs, stealth, Shadow DOM) - Impressum check (24 checks) - Deep consent verification (DataLayer, GCM, TCF) - CMP E2E tests (39 tests) - HTML email reports, FAQ, persistent history Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-05-11 11:44:20 +02:00
parent e80bbe000f 2f0f76e365
commit 36c6101b91
175 changed files with 20063 additions and 1283 deletions
@@ -40,107 +40,8 @@ class ScanResult:
    missing_pages: dict = field(default_factory=dict)  # url -> status_code


-# ── Service Registry ──────────────────────────────────────────────────────────
-# Each entry: regex pattern -> service metadata
-SERVICE_REGISTRY: dict[str, dict] = {
-    # --- Tracking & Analytics ---
-    r"google.?analytics|gtag\(|UA-\d+|G-\w{5,}": {
-        "id": "google_analytics", "name": "Google Analytics", "category": "tracking",
-        "provider": "Google LLC", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "Art. 44-49 DSGVO, §25 TDDDG",
-    },
-    r"googletagmanager|gtm\.js": {
-        "id": "google_tag_manager", "name": "Google Tag Manager", "category": "tracking",
-        "provider": "Google LLC", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "Art. 44-49 DSGVO",
-    },
-    r"facebook\.net/.*fbevents|fbq\(": {
-        "id": "facebook_pixel", "name": "Meta/Facebook Pixel", "category": "marketing",
-        "provider": "Meta Platforms", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "Art. 44-49 DSGVO, §25 TDDDG",
-    },
-    r"hotjar\.com|_hjSettings": {
-        "id": "hotjar", "name": "Hotjar", "category": "tracking",
-        "provider": "Hotjar Ltd", "country": "MT", "eu_adequate": True,
-        "requires_consent": True, "legal_ref": "§25 TDDDG (Session Recording)",
-    },
-    r"clarity\.ms": {
-        "id": "ms_clarity", "name": "Microsoft Clarity", "category": "tracking",
-        "provider": "Microsoft", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "§25 TDDDG (Session Replay), Art. 44 DSGVO",
-    },
-    r"matomo|piwik": {
-        "id": "matomo", "name": "Matomo", "category": "tracking",
-        "provider": "InnoCraft/Self-hosted", "country": "EU/Self", "eu_adequate": True,
-        "requires_consent": False, "legal_ref": "Cookieless moeglich, §25 TDDDG",
-    },
-    r"plausible\.io": {
-        "id": "plausible", "name": "Plausible Analytics", "category": "tracking",
-        "provider": "Plausible Insights", "country": "EE", "eu_adequate": True,
-        "requires_consent": False, "legal_ref": "EU-Anbieter, cookieless",
-    },
-    # --- CDN & Fonts ---
-    r"fonts\.googleapis\.com|fonts\.gstatic\.com": {
-        "id": "google_fonts", "name": "Google Fonts (remote)", "category": "cdn",
-        "provider": "Google LLC", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "LG Muenchen I, Az. 3 O 17493/20",
-    },
-    r"cdn\.cloudflare\.com|cdnjs\.cloudflare\.com": {
-        "id": "cloudflare_cdn", "name": "Cloudflare CDN", "category": "cdn",
-        "provider": "Cloudflare Inc", "country": "US", "eu_adequate": False,
-        "requires_consent": False, "legal_ref": "Art. 44-49 DSGVO, berechtigtes Interesse",
-    },
-    # --- Chatbots ---
-    r"widget\.intercom\.io|intercomcdn": {
-        "id": "intercom", "name": "Intercom", "category": "chatbot",
-        "provider": "Intercom Inc", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "Art. 44-49 DSGVO, KI-gestuetzt",
-    },
-    r"tidio\.co|tidioChatApi": {
-        "id": "tidio", "name": "Tidio Chat", "category": "chatbot",
-        "provider": "Tidio LLC", "country": "PL", "eu_adequate": True,
-        "requires_consent": False, "legal_ref": "EU-Anbieter",
-    },
-    r"zendesk\.com/embeddable|zdassets": {
-        "id": "zendesk", "name": "Zendesk", "category": "chatbot",
-        "provider": "Zendesk Inc", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "Art. 44-49 DSGVO",
-    },
-    # --- Payment ---
-    r"js\.stripe\.com|stripe\.com/v3": {
-        "id": "stripe", "name": "Stripe", "category": "payment",
-        "provider": "Stripe Inc", "country": "US", "eu_adequate": False,
-        "requires_consent": False, "legal_ref": "Art. 6(1)(b) Vertragserfuellung, SCCs",
-    },
-    r"paypal\.com/sdk|paypalobjects": {
-        "id": "paypal", "name": "PayPal", "category": "payment",
-        "provider": "PayPal Holdings", "country": "US", "eu_adequate": False,
-        "requires_consent": False, "legal_ref": "Art. 6(1)(b) Vertragserfuellung",
-    },
-    r"klarna\.com|klarna-payments": {
-        "id": "klarna", "name": "Klarna", "category": "payment",
-        "provider": "Klarna AB", "country": "SE", "eu_adequate": True,
-        "requires_consent": False, "legal_ref": "EU, aber Art. 22 DSGVO bei Bonitaetspruefung!",
-    },
-    # --- Captcha ---
-    r"recaptcha|grecaptcha": {
-        "id": "recaptcha", "name": "Google reCAPTCHA", "category": "other",
-        "provider": "Google LLC", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "Art. 44-49 DSGVO, §25 TDDDG",
-    },
-    # --- Video ---
-    r"youtube\.com/embed|youtube-nocookie|ytimg": {
-        "id": "youtube", "name": "YouTube", "category": "other",
-        "provider": "Google LLC", "country": "US", "eu_adequate": False,
-        "requires_consent": True, "legal_ref": "Art. 44-49 DSGVO, 2-Klick empfohlen",
-    },
-    # --- Consent Management ---
-    r"didomi|cookiebot|onetrust|usercentrics|consentmanager|quantcast": {
-        "id": "cmp", "name": "Consent Management Platform", "category": "other",
-        "provider": "Various", "country": "EU", "eu_adequate": True,
-        "requires_consent": False, "legal_ref": "CMP vorhanden — gut",
-    },
-}
+# ── Service Registry (imported from master) ──────────────────────────────────
+from compliance.services.service_registry import SERVICE_REGISTRY  # noqa: E402

 AI_TEXT_PATTERNS = [
    r"k(?:ue|ü)nstliche.?intelligenz",
@@ -157,9 +58,13 @@ AI_TEXT_PATTERNS = [

 FOOTER_LINK_PATTERNS = [
    (r'href="([^"]*(?:impressum|imprint|legal-notice)[^"]*)"', "impressum"),
-    (r'href="([^"]*(?:datenschutz|privacy|dsgvo)[^"]*)"', "datenschutz"),
+    (r'href="([^"]*(?:datenschutz|privacy|dsgvo|hinweise.?zum.?datenschutz)[^"]*)"', "datenschutz"),
    (r'href="([^"]*(?:agb|terms|nutzungsbedingungen)[^"]*)"', "agb"),
    (r'href="([^"]*(?:cookie)[^"]*)"', "cookies"),
+    # Deep DSE links (regional pages, sub-pages, service marks)
+    (r'href="([^"]*(?:datenschutzinformation|datenschutzerklaerung|datenschutzerkl)[^"]*)"', "datenschutz_deep"),
+    # Navigation links often contain DSB/privacy sub-pages
+    (r'href="([^"]*(?:ueber.?uns.*datenschutz|servicemarken.*datenschutz|kontakt.*datenschutz)[^"]*)"', "datenschutz_nav"),
 ]


@@ -183,15 +88,46 @@ async def scan_website(base_url: str) -> ScanResult:
                href = match.group(1)
                if href.startswith("/"):
                    href = urljoin(origin, href)
-                if href.startswith(origin):
+                if href.startswith(origin) and not re.search(r"\.(css|js|png|jpg|gif|svg|pdf|zip)(\?|$)", href):
                    page_urls.add(href)

-        # 3. Scan all pages (max 10)
-        for url in list(page_urls)[:10]:
-            html = start_html if url == origin else await _fetch_page(client, url, result)
-            if html:
+        # 3. Scan all pages in PARALLEL (max 10)
+        import asyncio
+        other_urls = [u for u in list(page_urls)[:10] if u != origin]
+        fetch_tasks = [_fetch_page(client, u, result) for u in other_urls]
+        other_htmls = await asyncio.gather(*fetch_tasks, return_exceptions=True)
+
+        # Process start page
+        _detect_services(start_html, origin, result)
+        _detect_ai_mentions(start_html, origin, result)
+
+        # Process other pages + discover DSE-internal links
+        dse_internal_urls = set()
+        for url, html in zip(other_urls, other_htmls):
+            if isinstance(html, str) and html:
                _detect_services(html, url, result)
                _detect_ai_mentions(html, url, result)
+                # If this is a DSE page, find links within it (SAME DOMAIN only)
+                if re.search(r"datenschutz|privacy|dsgvo", url, re.IGNORECASE):
+                    for pattern, _ in FOOTER_LINK_PATTERNS:
+                        for match in re.finditer(pattern, html, re.IGNORECASE):
+                            href = match.group(1)
+                            if href.startswith("/"):
+                                href = urljoin(origin, href)
+                            # IMPORTANT: Only follow links on the SAME domain
+                            # External links (etracker.com, google.de) must NOT be scanned
+                            if href.startswith(origin) and href not in page_urls:
+                                dse_internal_urls.add(href)
+
+        # 4. Follow DSE-internal links (additional pages linked from privacy policy)
+        if dse_internal_urls:
+            extra_urls = [u for u in list(dse_internal_urls)[:5] if u not in page_urls]
+            if extra_urls:
+                extra_tasks = [_fetch_page(client, u, result) for u in extra_urls]
+                extra_htmls = await asyncio.gather(*extra_tasks, return_exceptions=True)
+                for url, html in zip(extra_urls, extra_htmls):
+                    if isinstance(html, str) and html:
+                        _detect_services(html, url, result)

    # Deduplicate services
    seen = set()