fix: 4 bugs from IHK Konstanz scan validation

1. DSE-Matcher: Google/YouTube false match — now requires 2+ word match
   for provider-name fallback, not just "Google" matching YouTube section
2. AGB/Widerrufsbelehrung: only_ecommerce flag — skips for non-shop
   websites (detected via payment providers, cart keywords)
3. DSE-internal link following — scanner now discovers links WITHIN the
   privacy policy and scans those too (finds regional DSE sub-pages)
4. Expanded keyword synonyms for DSE mandatory checks:
   - "Zweck und Rechtsgrundlage" now matches "zwecke"
   - "behoerdlichen datenschutzbeauftragt" matches DSB
   - "aufsichtsbehörde" with umlaut matches

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

backend-compliance/compliance/services/dse_matcher.py

@@ -64,8 +64,21 @@ def match_service_to_dse(
         )
 
     # Step 2: Search for provider name (e.g., "Google" for "Google Analytics")
+    # But only if the provider name is specific enough — avoid "Google" matching YouTube
     provider = service_name.split()[0] if " " in service_name else service_name
+    if len(provider) < 4 or provider.lower() in ("the", "a", "an"):
+        provider = service_name  # Too short/generic, use full name
+
     section = find_section_by_content(sections, provider)
+    # Verify: the section must actually be about THIS service, not just mention the provider
+    if section and provider.lower() != service_name.lower():
+        # Check if the full service name or a close variant is in the section
+        content_lower = section.content.lower()
+        service_words = service_name.lower().split()
+        # At least 2 words of the service name must match (not just "Google")
+        matching_words = sum(1 for w in service_words if w in content_lower)
+        if matching_words < 2 and service_name.lower() not in content_lower:
+            section = None  # False match — provider name found but wrong context
 
     if section:
         original = _extract_relevant_paragraph(section.content, provider)

backend-compliance/compliance/services/mandatory_content_checker.py

@@ -51,6 +51,7 @@ MANDATORY_DOCUMENTS = [
         "legal_ref": "§305 BGB (bei Vertragsschluss)",
         "patterns": [r"agb", r"nutzungsbedingung", r"terms"],
         "severity": "MEDIUM",
+        "only_ecommerce": True,  # Nur bei Shops/Buchungsseiten
     },
     {
         "id": "widerruf",
@@ -58,6 +59,7 @@ MANDATORY_DOCUMENTS = [
         "legal_ref": "§355 BGB, Art. 246a §1 EGBGB (nur Fernabsatz)",
         "patterns": [r"widerruf", r"cancellation.?policy", r"right.?of.?withdrawal"],
         "severity": "MEDIUM",
+        "only_ecommerce": True,  # Nur bei Fernabsatzvertraegen
     },
 ]
 
@@ -78,21 +80,27 @@ MANDATORY_DSE_CONTENT = [
         "id": "dsb_kontakt",
         "name": "Kontaktdaten des Datenschutzbeauftragten",
         "legal_ref": "Art. 13 Abs. 1 lit. b DSGVO",
-        "keywords": ["datenschutzbeauftragt", "data protection officer", "dsb", "dpo"],
+        "keywords": ["datenschutzbeauftragt", "data protection officer", "dsb", "dpo",
+                      "behördlichen datenschutz", "behoerdlichen datenschutz",
+                      "datenschutz@", "datenschutzbeauftragter"],
         "severity": "HIGH",
     },
     {
         "id": "zwecke",
         "name": "Zwecke der Datenverarbeitung",
         "legal_ref": "Art. 13 Abs. 1 lit. c DSGVO",
-        "keywords": ["zweck", "purpose", "verarbeitungszweck"],
+        "keywords": ["zweck", "purpose", "verarbeitungszweck", "verarbeitungszwecke",
+                      "wozu", "wofuer", "zu welchem zweck", "nutzungszweck",
+                      "zweck und rechtsgrundlage", "zwecke der verarbeitung"],
         "severity": "HIGH",
     },
     {
         "id": "rechtsgrundlage",
         "name": "Rechtsgrundlagen der Verarbeitung",
         "legal_ref": "Art. 13 Abs. 1 lit. c DSGVO",
-        "keywords": ["rechtsgrundlage", "legal basis", "art. 6", "art.6"],
+        "keywords": ["rechtsgrundlage", "legal basis", "art. 6", "art.6",
+                      "berechtigtes interesse", "einwilligung", "vertragserfuellung",
+                      "vertragserfüllung", "rechtliche verpflichtung"],
         "severity": "HIGH",
     },
     {
@@ -116,8 +124,9 @@ MANDATORY_DSE_CONTENT = [
         "id": "beschwerderecht",
         "name": "Beschwerderecht bei Aufsichtsbehoerde",
         "legal_ref": "Art. 13 Abs. 2 lit. d DSGVO",
-        "keywords": ["aufsichtsbehoerde", "beschwerde", "supervisory authority",
+        "keywords": ["aufsichtsbehoerde", "aufsichtsbehörde", "beschwerde",
-                      "datenschutzbehoerde"],
+                      "supervisory authority", "datenschutzbehoerde",
+                      "landesbeauftragte", "bundesdatenschutz", "bfdi"],
         "severity": "MEDIUM",
     },
     {
@@ -183,13 +192,32 @@ MANDATORY_IMPRESSUM_CONTENT = [
 ]
 
 
+ECOMMERCE_INDICATORS = [
+    r"warenkorb", r"cart", r"shop", r"bestell", r"order",
+    r"checkout", r"kasse", r"buy", r"kaufen", r"add.?to.?cart",
+    r"stripe|paypal|klarna|mollie|adyen",  # Payment providers
+]
+
+
+def _is_ecommerce(scanned_pages: list[str], html_content: str = "") -> bool:
+    """Detect if website is an e-commerce/transactional site."""
+    all_text = " ".join(scanned_pages).lower() + " " + html_content.lower()
+    return any(re.search(p, all_text) for p in ECOMMERCE_INDICATORS)
+
+
 def check_mandatory_documents(
     scanned_pages: list[str], page_status: dict[str, int],
+    html_content: str = "",
 ) -> list[MandatoryFinding]:
     """Check if mandatory documents/pages exist on the website."""
     findings = []
+    is_shop = _is_ecommerce(scanned_pages, html_content)
 
     for doc in MANDATORY_DOCUMENTS:
+        # Skip e-commerce-only checks for non-shop websites
+        if doc.get("only_ecommerce") and not is_shop:
+            continue
+
         found = False
         for page in scanned_pages:
             if any(re.search(p, page, re.IGNORECASE) for p in doc["patterns"]):

backend-compliance/compliance/services/website_scanner.py

@@ -58,9 +58,11 @@ AI_TEXT_PATTERNS = [
 
 FOOTER_LINK_PATTERNS = [
     (r'href="([^"]*(?:impressum|imprint|legal-notice)[^"]*)"', "impressum"),
-    (r'href="([^"]*(?:datenschutz|privacy|dsgvo)[^"]*)"', "datenschutz"),
+    (r'href="([^"]*(?:datenschutz|privacy|dsgvo|hinweise.?zum.?datenschutz)[^"]*)"', "datenschutz"),
     (r'href="([^"]*(?:agb|terms|nutzungsbedingungen)[^"]*)"', "agb"),
     (r'href="([^"]*(?:cookie)[^"]*)"', "cookies"),
+    # Also find deep DSE links (regional pages, sub-pages)
+    (r'href="([^"]*(?:datenschutzinformation|datenschutzerklaerung|datenschutzerkl)[^"]*)"', "datenschutz_deep"),
 ]
 
 
@@ -97,11 +99,31 @@ async def scan_website(base_url: str) -> ScanResult:
         _detect_services(start_html, origin, result)
         _detect_ai_mentions(start_html, origin, result)
 
-        # Process other pages
+        # Process other pages + discover DSE-internal links
+        dse_internal_urls = set()
         for url, html in zip(other_urls, other_htmls):
             if isinstance(html, str) and html:
                 _detect_services(html, url, result)
                 _detect_ai_mentions(html, url, result)
+                # If this is a DSE page, find links within it
+                if re.search(r"datenschutz|privacy|dsgvo", url, re.IGNORECASE):
+                    for pattern, _ in FOOTER_LINK_PATTERNS:
+                        for match in re.finditer(pattern, html, re.IGNORECASE):
+                            href = match.group(1)
+                            if href.startswith("/"):
+                                href = urljoin(origin, href)
+                            if href.startswith("http") and href not in page_urls:
+                                dse_internal_urls.add(href)
+
+        # 4. Follow DSE-internal links (additional pages linked from privacy policy)
+        if dse_internal_urls:
+            extra_urls = [u for u in list(dse_internal_urls)[:5] if u not in page_urls]
+            if extra_urls:
+                extra_tasks = [_fetch_page(client, u, result) for u in extra_urls]
+                extra_htmls = await asyncio.gather(*extra_tasks, return_exceptions=True)
+                for url, html in zip(extra_urls, extra_htmls):
+                    if isinstance(html, str) and html:
+                        _detect_services(html, url, result)
 
     # Deduplicate services
     seen = set()