refactor(agent-check): split routes file (2692→347 LOC) + wire B1/B3/A1 [guardrail-change]
Phase-5 split of agent_compliance_check_routes.py — the 2700-line
monolith was decomposed into 19 modules in compliance/api/agent_check/:
- Phase A-F: resolve / profile+check / banner+TCF / vendors raw+finalize /
HTML blocks top+mid+bot / email / persist
- Helpers: _constants, _helpers, _fetch, _discovery, _single_check
- Schemas + State + thin _orchestrator
A1 ZIP-Anhang nativ in _phase_e_email: evidence_zip_builder.py bundles
slices + manifest.json + audit_metadata.json (SHA256 per slice +
build_sha + source_url). smtp_sender.py erweitert um attachments-Parameter.
B1 COOKIE-CONSENT-UX-001 (Mobile Reachability): consent_reachability_check.py
parses footer anchors, classifies intent (reopen_cmp / info_only /
browser_deflect) + target (same_page_cmp / new_tab / external).
_b1_wiring.py fetches homepage with iPhone-UA + renders Art-7-Abs-3
severity-coloured block.
B3 TH-RETENTION (Cross-Doc Speicherdauer): retention_comparator.py
compares DSI claim ↔ cookie-table duration ↔ actual Max-Age/expires
with 5% tolerance + severity hierarchy (dsi_under_actual HIGH,
table_under_actual HIGH, dsi_vs_table MEDIUM, actual_under_table LOW
Safari-ITP-Hint). _b3_wiring.py + Top-10 mismatches table in mail.
Side-effects:
- Fixed silent UnboundLocalError in original Step 5 (gf_one_pager used
audit_quality_findings before declaration, caught by surrounding
except → block never rendered). New _phase_d3_blocks_bot.py runs
audit-quality FIRST.
- agent_compliance_check_routes.py removed from loc-exceptions.txt
("Phase 5 split target" — done).
Tests: 55/55 grün (B1 22 + B3 27 + saving_scan 6).
E2E: smoke against Elli DSE+Cookie produced HIGH/missing B1 finding,
TH-RETENTION table (17 cookies / 3 ✓ / 3 ✗ / 11 ?), evidence-zip
with 2 slices + manifest + audit_metadata (12089B, SHA256-chained,
source verified), email sent (attachments=1).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
@@ -0,0 +1,10 @@
|
||||
"""
|
||||
Subpackage for the compliance-check route — extracted to keep
|
||||
`agent_compliance_check_routes.py` under the 500-line guardrail.
|
||||
|
||||
The route module still owns the public HTTP endpoints and re-exports
|
||||
all helpers from this subpackage, so external callers
|
||||
(`saving_scan_routes`, `agent_migration_routes`, tests) continue to
|
||||
import them from `compliance.api.agent_compliance_check_routes`
|
||||
unchanged.
|
||||
"""
|
||||
@@ -0,0 +1,105 @@
|
||||
"""B1 wiring — Mobile Consent-Reachability check + HTML block.
|
||||
|
||||
Fetches the homepage of the first submitted URL, runs the static
|
||||
`evaluate_reachability` analysis on the footer, and renders the
|
||||
result as an HTML block for the audit mail.
|
||||
|
||||
Only renders a block when the check FAILS — a passing site doesn't
|
||||
need a block. The block is severity-colored and lists the specific
|
||||
notes that triggered the finding (missing reopen anchor, new-tab
|
||||
break, browser-deflection language).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from compliance.services.consent_reachability_check import (
|
||||
evaluate_reachability,
|
||||
)
|
||||
|
||||
from ._helpers import _update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_b1(state: dict) -> None:
|
||||
"""Run the reachability check + render HTML. Mutates state in place."""
|
||||
req = state["req"]
|
||||
check_id = state["check_id"]
|
||||
homepage_url = ""
|
||||
for d in req.documents:
|
||||
if d.url:
|
||||
from urllib.parse import urlparse
|
||||
p = urlparse(d.url)
|
||||
if p.scheme and p.netloc:
|
||||
homepage_url = f"{p.scheme}://{p.netloc}/"
|
||||
break
|
||||
if not homepage_url:
|
||||
return
|
||||
|
||||
_update(check_id, "Mobile Consent-Reachability prüfen...", 95)
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=20.0, follow_redirects=True,
|
||||
headers={"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 "
|
||||
"like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) "
|
||||
"Version/17.5 Mobile/15E148 Safari/604.1"},
|
||||
) as c:
|
||||
r = await c.get(homepage_url)
|
||||
if r.status_code != 200:
|
||||
logger.info("B1: homepage fetch %s → HTTP %d", homepage_url, r.status_code)
|
||||
return
|
||||
page_html = r.text
|
||||
except Exception as e:
|
||||
logger.warning("B1: homepage fetch failed: %s", e)
|
||||
return
|
||||
|
||||
finding = evaluate_reachability(page_html, homepage_url)
|
||||
state["reachability_finding"] = finding
|
||||
state["reachability_html"] = _render_block(finding)
|
||||
logger.info(
|
||||
"B1 Reachability: passed=%s severity=%s reason=%s",
|
||||
finding["passed"], finding.get("severity"),
|
||||
finding.get("severity_reason"),
|
||||
)
|
||||
|
||||
|
||||
def _render_block(finding: dict) -> str:
|
||||
"""Render the reachability finding as an audit-mail HTML block."""
|
||||
if finding["passed"]:
|
||||
return ""
|
||||
sev = (finding.get("severity") or "").upper()
|
||||
color = "#dc2626" if sev == "HIGH" else "#f59e0b"
|
||||
notes_html = "".join(
|
||||
f"<li>{html.escape(n)}</li>" for n in finding.get("notes") or []
|
||||
)
|
||||
anchor = finding.get("reopen_anchor") or {}
|
||||
anchor_html = ""
|
||||
if anchor:
|
||||
anchor_html = (
|
||||
"<p style='margin:8px 0 0;font-size:13px;color:#475569;'>"
|
||||
"Gefundener Footer-Link: "
|
||||
f"<code>{html.escape((anchor.get('text') or '')[:80])}</code> "
|
||||
f"→ <code>{html.escape((anchor.get('href') or '')[:120])}</code> "
|
||||
f"(target_class: {html.escape(anchor.get('target_class') or '—')})"
|
||||
"</p>"
|
||||
)
|
||||
return (
|
||||
f"<div style='margin:24px 0;padding:16px;border-left:4px solid {color};"
|
||||
"background:#fef2f2;border-radius:4px;'>"
|
||||
f"<h2 style='margin:0 0 8px;color:{color};font-size:16px;'>"
|
||||
"COOKIE-CONSENT-UX-001 — Mobile Consent-Reachability</h2>"
|
||||
f"<p style='margin:0 0 8px;font-size:14px;'><strong>Severity:</strong> "
|
||||
f"{sev} ({html.escape(finding.get('severity_reason') or '')})</p>"
|
||||
"<p style='margin:0 0 4px;font-size:14px;'>"
|
||||
"Art. 7 Abs. 3 DSGVO: Widerruf muss so einfach wie Erteilung sein. "
|
||||
"Auf Mobile-Safari konnten wir folgendes Problem feststellen:</p>"
|
||||
f"<ul style='margin:8px 0 0 20px;font-size:14px;color:#7f1d1d;'>"
|
||||
f"{notes_html}</ul>"
|
||||
f"{anchor_html}"
|
||||
"</div>"
|
||||
)
|
||||
@@ -0,0 +1,189 @@
|
||||
"""B3 wiring — Cross-doc retention consistency check + HTML block.
|
||||
|
||||
Combines three sources of retention truth per cookie:
|
||||
|
||||
- DSI text (state["doc_texts"]["dse"] or "cookie")
|
||||
- cookie-table `duration` from cmp_vendors[i]["cookies"][j]
|
||||
- actual cookie expiry from banner_result["cookies_detailed"][k]
|
||||
|
||||
and produces per-cookie findings + a TH-RETENTION theme summary. Only
|
||||
renders an HTML block when there are findings to show; the block is
|
||||
sorted by severity (HIGH first) and shows the top-10 mismatches.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import time
|
||||
|
||||
from compliance.services.retention_comparator import (
|
||||
build_retention_theme_summary,
|
||||
compare_retention,
|
||||
extract_retention_claims,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _actual_max_age_seconds(cookie: dict) -> float | None:
|
||||
"""Get cookie Max-Age in seconds.
|
||||
|
||||
Playwright gives us `expires` as a Unix timestamp (seconds-since-
|
||||
epoch). Some sources give `max_age` directly. -1 / 0 means session
|
||||
cookie (no expiry) — return None to signal that.
|
||||
"""
|
||||
ma = cookie.get("max_age")
|
||||
if isinstance(ma, (int, float)) and ma > 0:
|
||||
return float(ma)
|
||||
exp = cookie.get("expires")
|
||||
if isinstance(exp, (int, float)) and exp > 0:
|
||||
delta = exp - time.time()
|
||||
if delta > 0:
|
||||
return float(delta)
|
||||
return None
|
||||
|
||||
|
||||
def run_b3(state: dict) -> None:
|
||||
"""Cross-doc retention check + render HTML. Mutates state in place."""
|
||||
doc_texts = state["doc_texts"]
|
||||
cmp_vendors = state["cmp_vendors"]
|
||||
banner_result = state["banner_result"]
|
||||
|
||||
dsi_text = doc_texts.get("dse") or doc_texts.get("cookie") or ""
|
||||
if not dsi_text:
|
||||
return
|
||||
|
||||
cookie_records: list[dict] = []
|
||||
cookie_names: list[str] = []
|
||||
vendor_names: list[str] = []
|
||||
for v in cmp_vendors or []:
|
||||
vname = (v.get("name") or "").strip()
|
||||
if vname:
|
||||
vendor_names.append(vname)
|
||||
for c in (v.get("cookies") or []):
|
||||
cname = (c.get("name") or "").strip()
|
||||
if not cname:
|
||||
continue
|
||||
duration = (c.get("duration") or c.get("persistence")
|
||||
or c.get("expiry") or "")
|
||||
cookie_names.append(cname)
|
||||
cookie_records.append({
|
||||
"name": cname,
|
||||
"vendor": vname,
|
||||
"table_duration": duration,
|
||||
"actual_max_age": None,
|
||||
})
|
||||
|
||||
if not cookie_records:
|
||||
return
|
||||
|
||||
# Match actual max_age from banner_result.cookies_detailed
|
||||
if banner_result:
|
||||
cookies_detailed = banner_result.get("cookies_detailed") or []
|
||||
by_name: dict[str, dict] = {}
|
||||
for c in cookies_detailed:
|
||||
n = (c.get("name") or "").lower()
|
||||
if n:
|
||||
by_name[n] = c
|
||||
for rec in cookie_records:
|
||||
nm = rec["name"].lower()
|
||||
if nm in by_name:
|
||||
rec["actual_max_age"] = _actual_max_age_seconds(by_name[nm])
|
||||
|
||||
claims = extract_retention_claims(dsi_text, cookie_names, vendor_names)
|
||||
|
||||
findings: list[dict] = []
|
||||
for rec in cookie_records:
|
||||
finding = compare_retention(
|
||||
cookie_name=rec["name"],
|
||||
table_duration=rec["table_duration"],
|
||||
actual_max_age_seconds=rec["actual_max_age"],
|
||||
dsi_claims=claims,
|
||||
vendor_name=rec["vendor"] or None,
|
||||
)
|
||||
findings.append(finding)
|
||||
|
||||
summary = build_retention_theme_summary(findings)
|
||||
state["retention_findings"] = findings
|
||||
state["retention_theme_summary"] = summary
|
||||
state["retention_html"] = _render_block(summary, findings)
|
||||
logger.info(
|
||||
"B3 Retention: %d findings, %d passed, %d failed, %d incomplete",
|
||||
summary["total"], summary["passed"], summary["failed"],
|
||||
summary["incomplete"],
|
||||
)
|
||||
|
||||
|
||||
def _fmt_days(d: float | None) -> str:
|
||||
if d is None:
|
||||
return "—"
|
||||
if d < 1:
|
||||
return f"{int(d * 24)}h"
|
||||
if d < 30:
|
||||
return f"{int(d)}d"
|
||||
if d < 365:
|
||||
return f"{int(d / 30)}mo"
|
||||
return f"{d / 365:.1f}y"
|
||||
|
||||
|
||||
def _render_block(summary: dict, findings: list[dict]) -> str:
|
||||
if summary["total"] == 0:
|
||||
return ""
|
||||
failed_findings = [f for f in findings if not f.get("matches")
|
||||
and f.get("severity_reason") != "incomplete"]
|
||||
if not failed_findings:
|
||||
return "" # all OK, no block needed
|
||||
# Sort by severity (HIGH first) then diff_days desc
|
||||
sev_rank = {"HIGH": 0, "MEDIUM": 1, "LOW": 2}
|
||||
failed_findings.sort(key=lambda f: (
|
||||
sev_rank.get((f.get("severity") or "").upper(), 9),
|
||||
-(f.get("diff_days") or 0),
|
||||
))
|
||||
rows = []
|
||||
for f in failed_findings[:10]:
|
||||
sev = (f.get("severity") or "").upper()
|
||||
color = ("#dc2626" if sev == "HIGH"
|
||||
else "#f59e0b" if sev == "MEDIUM" else "#64748b")
|
||||
rows.append(
|
||||
"<tr>"
|
||||
f"<td style='padding:6px 10px;border-bottom:1px solid #e5e7eb;'>"
|
||||
f"<code>{html.escape(f.get('cookie_name') or '—')}</code></td>"
|
||||
f"<td style='padding:6px 10px;border-bottom:1px solid #e5e7eb;'>"
|
||||
f"{html.escape((f.get('vendor_name') or '—'))}</td>"
|
||||
f"<td style='padding:6px 10px;border-bottom:1px solid #e5e7eb;'>"
|
||||
f"DSI: {_fmt_days(f.get('dsi_days'))} • "
|
||||
f"Tabelle: {_fmt_days(f.get('table_days'))} • "
|
||||
f"Realität: {_fmt_days(f.get('actual_days'))}</td>"
|
||||
f"<td style='padding:6px 10px;border-bottom:1px solid #e5e7eb;"
|
||||
f"color:{color};font-weight:600;'>"
|
||||
f"{sev} ({html.escape(f.get('mismatch_type') or '—')})</td>"
|
||||
"</tr>"
|
||||
)
|
||||
total = summary["total"]
|
||||
passed = summary["passed"]
|
||||
failed = summary["failed"]
|
||||
incomplete = summary["incomplete"]
|
||||
return (
|
||||
"<div style='margin:24px 0;padding:16px;border-left:4px solid #dc2626;"
|
||||
"background:#fefce8;border-radius:4px;'>"
|
||||
"<h2 style='margin:0 0 8px;color:#854d0e;font-size:16px;'>"
|
||||
"TH-RETENTION — Speicherdauer-Konsistenz (DSI ↔ Cookie-Tabelle ↔ Realität)"
|
||||
"</h2>"
|
||||
"<p style='margin:0 0 8px;font-size:14px;color:#3f3f46;'>"
|
||||
f"<strong>{total}</strong> Cookies verglichen: "
|
||||
f"<strong style='color:#15803d;'>{passed} ✓</strong> / "
|
||||
f"<strong style='color:#dc2626;'>{failed} ✗</strong> / "
|
||||
f"<strong style='color:#64748b;'>{incomplete} ?</strong></p>"
|
||||
"<table style='width:100%;border-collapse:collapse;font-size:13px;"
|
||||
"margin-top:8px;background:#fff;'>"
|
||||
"<thead><tr style='background:#f1f5f9;'>"
|
||||
"<th style='text-align:left;padding:6px 10px;'>Cookie</th>"
|
||||
"<th style='text-align:left;padding:6px 10px;'>Vendor</th>"
|
||||
"<th style='text-align:left;padding:6px 10px;'>Werte</th>"
|
||||
"<th style='text-align:left;padding:6px 10px;'>Mismatch</th>"
|
||||
"</tr></thead>"
|
||||
f"<tbody>{''.join(rows)}</tbody>"
|
||||
"</table>"
|
||||
"</div>"
|
||||
)
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Module-level constants + shared job state for the compliance-check
|
||||
route.
|
||||
|
||||
`_compliance_check_jobs` is the SINGLE source of truth for in-flight
|
||||
job progress. Other modules MUST import the same object — never
|
||||
re-declare it — otherwise progress updates land in a detached dict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Internal hostname of the consent-tester container.
|
||||
CONSENT_TESTER_URL = "http://bp-compliance-consent-tester:8094"
|
||||
|
||||
# In-memory job registry. Keyed by check_id. Values:
|
||||
# {"status": "running"|"completed"|"failed"|"skipped_tdm",
|
||||
# "progress": str, "progress_pct": int, "result": dict, ...}
|
||||
# Read/written by:
|
||||
# - agent_compliance_check_routes (start/status/_run/_update)
|
||||
# - saving_scan_routes (start)
|
||||
# - agent_migration_routes (status mirror)
|
||||
_compliance_check_jobs: dict[str, dict] = {}
|
||||
|
||||
|
||||
# Canonical doc types in the same order the frontend
|
||||
# ComplianceCheckTab renders them. The route pads `results` to always
|
||||
# include an entry for each — missing rows are flagged as 'Nicht
|
||||
# eingereicht' or 'Auf der Website nicht gefunden'.
|
||||
#
|
||||
# DSB-Kontakt is NOT canonical: per GDPR practice the DSB is named
|
||||
# inside the DSI/datenschutz document (email or contact block), not as
|
||||
# a separate page. We check 'DSB benannt' as a sub-check of the DSE.
|
||||
_ALL_DOC_TYPES = [
|
||||
"dse", "impressum", "social_media", "cookie",
|
||||
"agb", "nutzungsbedingungen", "widerruf",
|
||||
]
|
||||
|
||||
|
||||
# Human-readable labels per doc_type. Used in the report + emails.
|
||||
_DOC_TYPE_LABELS = {
|
||||
"dse": "Datenschutzerklaerung",
|
||||
"datenschutz": "Datenschutzerklaerung",
|
||||
"privacy": "Datenschutzerklaerung",
|
||||
"impressum": "Impressum",
|
||||
"agb": "AGB",
|
||||
"widerruf": "Widerrufsbelehrung",
|
||||
"cookie": "Cookie-Richtlinie",
|
||||
"avv": "Auftragsverarbeitung",
|
||||
"loeschkonzept": "Loeschkonzept",
|
||||
"dsfa": "Datenschutz-Folgenabschaetzung",
|
||||
"social_media": "Social Media Datenschutz",
|
||||
"nutzungsbedingungen": "Nutzungsbedingungen",
|
||||
"dsb": "DSB-Kontakt",
|
||||
# P74: Legal-Notice / Rechtliche Hinweise (IP, Forward-Looking, Risiko)
|
||||
"legal_notice": "Rechtliche Hinweise",
|
||||
# P96: Digital Services Act-Pflichtangaben (Art. 12+17 DSA)
|
||||
"dsa": "DSA-Pflichtangaben",
|
||||
# P97: Lizenzhinweise Dritter (OSS-Compliance)
|
||||
"lizenzhinweise": "Lizenzhinweise Dritter",
|
||||
}
|
||||
|
||||
|
||||
# Title/URL keywords → canonical doc_type. Order matters: most-specific first.
|
||||
_DISCOVERY_RULES: list[tuple[str, tuple[str, ...]]] = [
|
||||
("cookie", ("cookie", "kuche", "biscuit", "cookies-")),
|
||||
("widerruf", ("widerruf", "rueckgabe", "rückgabe", "cancellation",
|
||||
"right-of-withdrawal", "ruecktritts", "rücktritts")),
|
||||
("social_media", ("social-media", "soziale-medien", "social_media",
|
||||
"social-media-policy")),
|
||||
# P23: 'terms-and-conditions' kann Allgemeine Geschaeftsbedingungen ODER
|
||||
# Nutzungsbedingungen meinen. Discovery-Funktion klassifiziert spaeter
|
||||
# praeziser per Titel + Inhalt. Hier nur Url-Hint:
|
||||
("agb", ("/agb", "geschaeftsbedingungen", "geschäftsbedingungen",
|
||||
"general-terms")),
|
||||
("nutzungsbedingungen", ("nutzungsbedingung", "nutzungsbedingungen",
|
||||
"terms-of-use", "terms-and-conditions",
|
||||
"nutzungsordnung", "terms-of-service",
|
||||
"allgemeine-nutzungsbedingungen")),
|
||||
("dsb", ("datenschutzbeauftragt", "data-protection-officer",
|
||||
"dpo-contact", "/dsb")),
|
||||
("impressum", ("impressum", "imprint", "legal-notice", "site-notice",
|
||||
"anbieterkennzeichnung", "legal-disclaimer-pool")),
|
||||
("dse", ("data-privacy", "datenschutz", "data-protection",
|
||||
"privacy-policy", "privacy-notice", "dsgvo",
|
||||
"data_privacy", "datenschutzinformation")),
|
||||
]
|
||||
|
||||
|
||||
# Compound TLDs that count as 2 labels when extracting the second-level
|
||||
# domain (e.g. shop.example.co.uk → 'example', not 'co').
|
||||
_COMPOUND_TLDS = {
|
||||
"co.uk", "co.jp", "co.nz", "co.kr", "co.za", "co.in",
|
||||
"com.au", "com.br", "com.mx", "com.tr", "com.sg",
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Auto-discovery of missing canonical doc-types.
|
||||
|
||||
For each canonical type the user did NOT submit, try to find it on the
|
||||
homepage of the URLs they DID submit. Also follow same-owner subdomains
|
||||
mentioned in the submitted text (BMW Group → bmwgroup.com etc.).
|
||||
|
||||
Discovered docs are classified by `_classify_discovered_doc` and merged
|
||||
back into `doc_entries`; entries that stayed empty get
|
||||
`discovery_attempted=True` so the padding step can differentiate
|
||||
"Nicht eingereicht" from "Auf der Website nicht gefunden".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from ._constants import _ALL_DOC_TYPES, CONSENT_TESTER_URL
|
||||
from ._helpers import _classify_discovered_doc, _update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _autodiscover_missing(
|
||||
check_id: str,
|
||||
doc_entries: list[dict],
|
||||
doc_texts: dict[str, str],
|
||||
url_text_cache: dict[str, str],
|
||||
) -> None:
|
||||
"""For each canonical doc_type the user did not submit, try to find
|
||||
the corresponding document on the homepage of the site they DID submit.
|
||||
|
||||
Modifies doc_entries in place: fills text/url/word_count and sets
|
||||
`auto_discovered=True`. Marks `discovery_attempted=True` on every
|
||||
missing entry (even when nothing was found) so the report can
|
||||
distinguish 'Nicht eingereicht' from 'Auf der Website nicht gefunden'.
|
||||
"""
|
||||
# VW-Fix: nur Doc-Types mit substantieller Text-Ausbeute zaehlen
|
||||
# als 'submitted'. Wenn der User eine URL eingegeben hat aber die
|
||||
# 404 liefert (VW cookie-richtlinie.html), oder der Crawler weniger
|
||||
# als 200 Zeichen extrahiert (SPA-Shell), als 'missing' behandeln
|
||||
# damit der Discovery-Pass alternative URLs probiert.
|
||||
_MIN_USEFUL_CHARS = 200
|
||||
submitted_types = {
|
||||
e["doc_type"] for e in doc_entries
|
||||
if len((e.get("text") or "").strip()) >= _MIN_USEFUL_CHARS
|
||||
}
|
||||
# Markiere die fehlgeschlagenen URL-Submissions damit der Discovery
|
||||
# ihre URL nicht erneut probiert (waere sinnlos).
|
||||
failed_urls: set[str] = {
|
||||
(e.get("url") or "").strip()
|
||||
for e in doc_entries
|
||||
if (e.get("url") or "").strip()
|
||||
and len((e.get("text") or "").strip()) < _MIN_USEFUL_CHARS
|
||||
}
|
||||
if failed_urls:
|
||||
logger.info(
|
||||
"VW-Fix: %d eingegebene URLs lieferten <%d Zeichen — Discovery "
|
||||
"soll Alternativen probieren: %s",
|
||||
len(failed_urls), _MIN_USEFUL_CHARS,
|
||||
", ".join(list(failed_urls)[:3]),
|
||||
)
|
||||
# Map alias types to canonical
|
||||
submitted_canon = {
|
||||
"dse" if t in ("datenschutz", "privacy") else t for t in submitted_types
|
||||
}
|
||||
# Missing = canonical types the user did NOT submit
|
||||
missing = set(_ALL_DOC_TYPES) - submitted_canon
|
||||
if not missing:
|
||||
return
|
||||
|
||||
# Pick the most common base (scheme://netloc) from submitted URLs.
|
||||
bases: dict[str, int] = {}
|
||||
for e in doc_entries:
|
||||
u = (e.get("url") or "").strip()
|
||||
if u and "://" in u:
|
||||
p = urlparse(u)
|
||||
base = f"{p.scheme}://{p.netloc}"
|
||||
bases[base] = bases.get(base, 0) + 1
|
||||
if not bases:
|
||||
# No submitted URL at all — nothing to crawl from. Add empty
|
||||
# placeholders (with discovery_attempted=False) so the padding
|
||||
# step renders them as 'Nicht eingereicht' (not 'Nicht gefunden').
|
||||
for dt in missing:
|
||||
doc_entries.append({
|
||||
"doc_type": dt, "url": "", "text": "", "word_count": 0,
|
||||
"auto_discovered": False, "discovery_attempted": False,
|
||||
})
|
||||
return
|
||||
|
||||
# Build crawl plan: primary base + any related domains mentioned in
|
||||
# the submitted texts that share the owner's SLD. Example: BMW Group
|
||||
# text mentions bmwgroup.com and bmwgroup.jobs in addition to bmw.de.
|
||||
primary_base = max(bases, key=bases.get) + "/"
|
||||
crawl_bases: list[str] = [primary_base]
|
||||
primary_netloc = urlparse(primary_base).netloc.lower().lstrip("www.")
|
||||
owner_token = primary_netloc.split(".")[0] # 'bmw'
|
||||
|
||||
if owner_token and len(owner_token) >= 3:
|
||||
domain_re = re.compile(
|
||||
r"https?://([a-z0-9][a-z0-9\-]*\.)*" + re.escape(owner_token)
|
||||
+ r"[a-z0-9\-]*\.[a-z]{2,}",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
seen_bases = {primary_base}
|
||||
for entry in doc_entries:
|
||||
text = entry.get("text") or ""
|
||||
for m in domain_re.finditer(text):
|
||||
p = urlparse(m.group(0))
|
||||
base = f"{p.scheme}://{p.netloc}/"
|
||||
base_netloc = p.netloc.lower().lstrip("www.")
|
||||
if base_netloc == primary_netloc:
|
||||
continue
|
||||
if base in seen_bases:
|
||||
continue
|
||||
seen_bases.add(base)
|
||||
crawl_bases.append(base)
|
||||
if len(crawl_bases) >= 3:
|
||||
break
|
||||
if len(crawl_bases) >= 3:
|
||||
break
|
||||
|
||||
_update(
|
||||
check_id,
|
||||
f"Suche fehlende Dokumente auf {', '.join(urlparse(b).netloc for b in crawl_bases)}...",
|
||||
18,
|
||||
)
|
||||
|
||||
discovered: list[dict] = []
|
||||
disc_payloads: list[dict] = []
|
||||
disc_cookie_texts: list[str] = []
|
||||
for base in crawl_bases:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=300.0) as client: # P90: 180s -> 300s
|
||||
resp = await client.post(
|
||||
f"{CONSENT_TESTER_URL}/dsi-discovery",
|
||||
json={"url": base, "max_documents": 15},
|
||||
timeout=300.0, # P90: 180s -> 300s
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.warning("auto-discovery: HTTP %d for %s",
|
||||
resp.status_code, base)
|
||||
continue
|
||||
body = resp.json()
|
||||
discovered.extend(body.get("documents", []) or [])
|
||||
disc_payloads.extend(body.get("cmp_payloads") or [])
|
||||
cmp_text = body.get("cmp_cookie_text") or ""
|
||||
if cmp_text:
|
||||
disc_cookie_texts.append(cmp_text)
|
||||
logger.info("auto-discovery on %s: %d docs, %d CMP payloads, "
|
||||
"cmp_cookie_text=%d words", base,
|
||||
len(body.get("documents", []) or []),
|
||||
len(body.get("cmp_payloads") or []),
|
||||
len(cmp_text.split()))
|
||||
except Exception as e:
|
||||
# P90: verbose exception fuer Diagnose
|
||||
logger.warning("auto-discovery failed for %s: %s (%s)",
|
||||
base, str(e) or "(empty)", type(e).__name__)
|
||||
|
||||
# Classify each discovered doc into a canonical doc_type
|
||||
by_type: dict[str, dict] = {}
|
||||
for d in discovered:
|
||||
title = (d.get("title") or "").lower()
|
||||
url = (d.get("url") or "").lower()
|
||||
wc = d.get("word_count") or 0
|
||||
if wc < 100:
|
||||
continue
|
||||
canon = _classify_discovered_doc(title, url)
|
||||
if canon and canon in missing and canon not in by_type:
|
||||
by_type[canon] = d
|
||||
|
||||
# Append/Update entry for every missing canonical type. Auto-discovered
|
||||
# ones get the text/URL filled; ungratched ones stay empty so the
|
||||
# padding step renders them as 'Auf der Website nicht gefunden'.
|
||||
# VW-Fix: wenn schon ein leerer entry existiert (URL gesetzt, aber
|
||||
# fetch hat 0/Mini-Text geliefert), in-place updaten statt duplizieren.
|
||||
filled = 0
|
||||
for dt in missing:
|
||||
existing = next((e for e in doc_entries
|
||||
if e.get("doc_type") == dt), None)
|
||||
new_entry: dict = existing if existing else {
|
||||
"doc_type": dt, "url": "", "text": "", "word_count": 0,
|
||||
"auto_discovered": False, "discovery_attempted": True,
|
||||
"cmp_payloads": [],
|
||||
}
|
||||
new_entry["discovery_attempted"] = True
|
||||
d = by_type.get(dt)
|
||||
if d:
|
||||
full = d.get("full_text") or d.get("text_preview") or ""
|
||||
# For cookie: prefer the CMP-reconstructed text when it's
|
||||
# substantially richer than the auto-discovered DOM extraction.
|
||||
# BMW homepage CMP yields ~1800 words of authoritative policy;
|
||||
# DOM extraction typically yields ~600 words of site chrome.
|
||||
if dt == "cookie" and disc_cookie_texts:
|
||||
cmp_merged = "\n\n".join(disc_cookie_texts)
|
||||
if len(cmp_merged.split()) > len(full.split()):
|
||||
logger.info(
|
||||
"cookie: using CMP-reconstructed text (%d words) "
|
||||
"instead of DOM (%d words)",
|
||||
len(cmp_merged.split()), len(full.split()),
|
||||
)
|
||||
full = cmp_merged
|
||||
if len(full.split()) >= 100:
|
||||
new_entry["text"] = full
|
||||
# Behalte die original URL als "rejected_url" damit Audit
|
||||
# zeigt 'X war 404, wir haben Y gefunden'.
|
||||
if existing and (existing.get("url") or "").strip() in failed_urls:
|
||||
new_entry["rejected_url"] = existing.get("url")
|
||||
new_entry["url"] = d.get("url", "")
|
||||
new_entry["word_count"] = len(full.split())
|
||||
new_entry["auto_discovered"] = True
|
||||
if dt == "cookie" and disc_payloads:
|
||||
new_entry["cmp_payloads"] = disc_payloads
|
||||
doc_texts[dt] = full
|
||||
filled += 1
|
||||
logger.info(
|
||||
"auto-discovered %s on %s: %s (%d words)%s",
|
||||
dt, base, d.get("url", "")[:80], new_entry["word_count"],
|
||||
" [REPLACED failed URL]" if existing else "",
|
||||
)
|
||||
if not existing:
|
||||
doc_entries.append(new_entry)
|
||||
|
||||
logger.info(
|
||||
"auto-discovery: filled %d/%d missing types from %s",
|
||||
filled, len(missing), base,
|
||||
)
|
||||
@@ -0,0 +1,142 @@
|
||||
"""URL → text fetch helper for the compliance-check pipeline.
|
||||
|
||||
Tries the consent-tester service first (Playwright, full JS render +
|
||||
CMP capture). On any failure or empty result, falls back to a direct
|
||||
HTTP GET with an identifiable User-Agent and per-domain rate limiting.
|
||||
|
||||
For cookie/dse/social_media doc types we cap discovery to 1 sub-page
|
||||
(the policy itself is authoritative). For Impressum/AGB/Widerruf and
|
||||
similar enterprise-split pages we follow up to 3 sub-pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re as _re
|
||||
|
||||
import httpx
|
||||
|
||||
from ._constants import CONSENT_TESTER_URL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _fetch_text(url: str, doc_type: str = "") -> tuple[str, list[dict]]:
|
||||
"""Fetch text from URL via consent-tester, with HTTP fallback.
|
||||
|
||||
Returns (text, cmp_payloads). cmp_payloads is the raw CMP JSON captured
|
||||
during navigation (ePaaS, OneTrust, …) — empty when no CMP fired or
|
||||
HTTP fallback was used. Backend turns payloads into structured vendor
|
||||
records for the VVT table in the email.
|
||||
"""
|
||||
# 1. Consent-tester (Playwright-based, full JS rendering).
|
||||
# max_documents depends on doc_type:
|
||||
# - cookie/dse/social_media: self-extract (often + CMP capture) is
|
||||
# authoritative, sub-pages dilute the policy text. max=1.
|
||||
# - impressum/agb/widerruf/nutzungsbedingungen/dsb: BMW & similar
|
||||
# enterprise sites split this across 3-4 short sub-pages
|
||||
# (Versicherungsvermittler, Aufsicht, Berufsrecht). max=3 follows
|
||||
# them. The 15s networkidle bail (dsi_helpers) keeps timing safe.
|
||||
short_extract_types = {"cookie", "dse", "datenschutz", "privacy", "social_media"}
|
||||
max_docs = 1 if (doc_type or "") in short_extract_types else 3
|
||||
try:
|
||||
# P90: 120s reicht nicht fuer BMW-Impressum (Auto-Discovery folgt
|
||||
# 3 Sub-Docs). 240s gibt Spielraum. Mercedes faellt aktuell mit
|
||||
# 120s auch oft an Akamai-Latenz.
|
||||
async with httpx.AsyncClient(timeout=240.0) as client:
|
||||
resp = await client.post(
|
||||
f"{CONSENT_TESTER_URL}/dsi-discovery",
|
||||
json={"url": url, "max_documents": max_docs},
|
||||
timeout=240.0,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
payload = resp.json()
|
||||
docs = payload.get("documents", [])
|
||||
cmp_payloads = payload.get("cmp_payloads") or []
|
||||
cmp_cookie_text = payload.get("cmp_cookie_text") or ""
|
||||
# D — wenn der consent-tester HTML-Tabellen aus dem DOM
|
||||
# extrahiert hat, in die cmp_payloads als "generic_table"
|
||||
# einschleusen damit das Backend sie via cookies_table_parser
|
||||
# verarbeiten kann.
|
||||
for doc in (docs or []):
|
||||
for tbl in (doc.get("tables") or []):
|
||||
if not tbl or len(tbl) < 3:
|
||||
continue
|
||||
cmp_payloads.append({
|
||||
"kind": "html_table",
|
||||
"url": doc.get("url", ""),
|
||||
"rows": tbl,
|
||||
})
|
||||
if docs:
|
||||
texts = []
|
||||
for doc in docs:
|
||||
t = doc.get("full_text", "") or doc.get("text_preview", "") or ""
|
||||
if t and len(t) > 50:
|
||||
texts.append(t)
|
||||
merged = "\n\n".join(texts)
|
||||
# For cookie/dse/social_media: when CMP reconstruction is
|
||||
# substantially richer than DOM extraction, use it. This
|
||||
# fixes the BMW case where DOM yields ~600 words of
|
||||
# navigation but the ePaaS payload reconstructs to ~1800
|
||||
# words of actual cookie policy.
|
||||
if (doc_type in short_extract_types
|
||||
and cmp_cookie_text
|
||||
and len(cmp_cookie_text.split()) > len(merged.split())):
|
||||
logger.info(
|
||||
"Preferring CMP-reconstructed text for %s on %s "
|
||||
"(%d words CMP vs %d words DOM)",
|
||||
doc_type, url,
|
||||
len(cmp_cookie_text.split()),
|
||||
len(merged.split()),
|
||||
)
|
||||
merged = cmp_cookie_text
|
||||
if merged and len(merged.split()) > 100:
|
||||
if len(texts) > 1:
|
||||
logger.info("Merged %d docs from %s (%d words)",
|
||||
len(texts), url, len(merged.split()))
|
||||
return merged, cmp_payloads
|
||||
# P90-Bug-Fix: auch wenn DSE-Text zu kurz fuer 100-Wort-
|
||||
# Schwelle ist, die captured CMP-Payloads NICHT verwerfen.
|
||||
# BMW-Bug: DSE liefert 10 Wort SPA-Shell, aber ePaaS-JSON
|
||||
# (393KB) wurde captured. Backend braucht die fuer
|
||||
# extract_vendors_from_payloads (VVT-Tabelle).
|
||||
if cmp_payloads:
|
||||
logger.info(
|
||||
"P90: keeping %d CMP payloads for %s despite "
|
||||
"short text (%d words) — HTTP fallback runs in parallel",
|
||||
len(cmp_payloads), url,
|
||||
len((merged or cmp_cookie_text).split()),
|
||||
)
|
||||
fallback_text = merged or cmp_cookie_text or ""
|
||||
return fallback_text, cmp_payloads
|
||||
except Exception as e:
|
||||
# P90: verbose exception fuer Diagnose (war vorher empty)
|
||||
logger.warning("Consent-tester fetch failed for %s: %s (%s)",
|
||||
url, str(e) or "(empty)", type(e).__name__)
|
||||
|
||||
# 2. Fallback: direct HTTP fetch (works for SSR pages like BMW).
|
||||
# P7: kenntlicher UA + per-Domain Rate-Limit.
|
||||
try:
|
||||
from compliance.services.compliance_user_agent import (
|
||||
default_request_headers, DomainRateLimiter,
|
||||
)
|
||||
async with httpx.AsyncClient(
|
||||
timeout=30.0, follow_redirects=True,
|
||||
headers=default_request_headers(),
|
||||
) as client:
|
||||
async with DomainRateLimiter(url):
|
||||
resp = await client.get(url)
|
||||
if resp.status_code == 200 and "text/html" in resp.headers.get("content-type", ""):
|
||||
html = resp.text
|
||||
# Strip HTML tags, decode entities
|
||||
text = _re.sub(r"<script[^>]*>.*?</script>", " ", html, flags=_re.DOTALL | _re.IGNORECASE)
|
||||
text = _re.sub(r"<style[^>]*>.*?</style>", " ", text, flags=_re.DOTALL | _re.IGNORECASE)
|
||||
text = _re.sub(r"<[^>]+>", " ", text)
|
||||
text = _re.sub(r"\s+", " ", text).strip()
|
||||
if len(text.split()) > 100:
|
||||
logger.info("HTTP fallback for %s: %d words", url, len(text.split()))
|
||||
return text, []
|
||||
except Exception as e:
|
||||
logger.warning("HTTP fallback failed for %s: %s", url, e)
|
||||
|
||||
return "", []
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Pure helpers for the compliance-check route — no I/O, no async.
|
||||
|
||||
Grouped here because each is small and they share the same constants
|
||||
imports. Splitting further would not improve readability.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ._constants import (
|
||||
_ALL_DOC_TYPES,
|
||||
_COMPOUND_TLDS,
|
||||
_DISCOVERY_RULES,
|
||||
_DOC_TYPE_LABELS,
|
||||
_compliance_check_jobs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _update(check_id: str, msg: str, pct: int | None = None) -> None:
|
||||
"""Update the in-memory job entry with a progress message + pct."""
|
||||
job = _compliance_check_jobs[check_id]
|
||||
job["progress"] = msg
|
||||
if pct is not None:
|
||||
job["progress_pct"] = max(0, min(100, int(pct)))
|
||||
|
||||
|
||||
def _doc_type_label(doc_type: str) -> str:
|
||||
return _DOC_TYPE_LABELS.get(doc_type, doc_type.upper())
|
||||
|
||||
|
||||
def _classify_discovered_doc(title: str, url: str) -> str | None:
|
||||
"""Map a discovered doc (by its title + URL) to one of our 8 canonical types."""
|
||||
haystack = f"{title} {url}"
|
||||
for canon, keywords in _DISCOVERY_RULES:
|
||||
if any(kw in haystack for kw in keywords):
|
||||
return canon
|
||||
return None
|
||||
|
||||
|
||||
def _extract_domain(doc_entries: list[dict]) -> str | None:
|
||||
"""Extract base domain (without www) from first URL."""
|
||||
for entry in doc_entries:
|
||||
url = entry.get("url", "")
|
||||
if url and "://" in url:
|
||||
host = urlparse(url).netloc.lower()
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
return host or None
|
||||
return None
|
||||
|
||||
|
||||
def _company_name_from_url(doc_entries: list[dict]) -> str | None:
|
||||
"""Derive a display company name from the entered URLs.
|
||||
|
||||
Heuristic: take the second-level domain (e.g. "bmw" from "www.bmw.de"),
|
||||
uppercase short acronyms (<=4 chars, no hyphens), title-case the rest.
|
||||
|
||||
Examples:
|
||||
www.bmw.de -> BMW
|
||||
mercedes-benz.de -> Mercedes-Benz
|
||||
shop.example.co.uk -> Example
|
||||
juris.de -> Juris
|
||||
"""
|
||||
for entry in doc_entries:
|
||||
url = entry.get("url", "")
|
||||
if not url or "://" not in url:
|
||||
continue
|
||||
host = urlparse(url).netloc.lower()
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
parts = host.split(".")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
# Handle compound TLDs (.co.uk etc.)
|
||||
if len(parts) >= 3 and ".".join(parts[-2:]) in _COMPOUND_TLDS:
|
||||
sld = parts[-3]
|
||||
else:
|
||||
sld = parts[-2]
|
||||
if not sld:
|
||||
continue
|
||||
if len(sld) <= 4 and "-" not in sld:
|
||||
return sld.upper()
|
||||
return "-".join(p.capitalize() for p in sld.split("-"))
|
||||
return None
|
||||
|
||||
|
||||
def _get_skip_types(profile) -> dict[str, str]:
|
||||
"""Doc_types to skip entirely with a per-type reason message.
|
||||
|
||||
Heute primaer fuer OEM-Konfigurator-Pattern (BMW/Audi/Mercedes):
|
||||
wenn die Site kein Direkt-Vertrieb macht, sind AGB/Widerruf/
|
||||
Nutzungsbedingungen nicht Pflicht auf der Website — sie werden
|
||||
beim Vertragshaendler ausgehaendigt.
|
||||
"""
|
||||
if getattr(profile, "no_direct_sales", False):
|
||||
msg = (
|
||||
"Nicht anwendbar — die Webseite schliesst keinen Direkt-"
|
||||
"Kaufvertrag (OEM-Konfigurator-Pattern, Vertrag laeuft "
|
||||
"ueber Vertragshaendler). AGB/Widerruf werden beim "
|
||||
"Haendler ausgehaendigt."
|
||||
)
|
||||
return {
|
||||
"agb": msg,
|
||||
"widerruf": msg,
|
||||
"nutzungsbedingungen": msg,
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def _apply_profile_filter(result, profile, doc_type: str):
|
||||
"""Adjust INFO-level checks based on business profile context.
|
||||
|
||||
For example: ODR check only relevant for B2C online shops.
|
||||
"""
|
||||
for check in result.checks:
|
||||
cid = check.id.lower()
|
||||
|
||||
# ODR/OS-Link: relevant ONLY for B2C online shops. The check's
|
||||
# default hint is written for B2B (it explains why it's not
|
||||
# relevant) — for B2C we must replace it with action-oriented
|
||||
# guidance, otherwise the report contradicts itself.
|
||||
if "odr" in cid or "os-link" in cid or "streitbeilegung" in check.label.lower():
|
||||
if profile.needs_odr:
|
||||
if not check.passed:
|
||||
check.hint = (
|
||||
"Als B2C-Anbieter muessen Sie nach Art. 14 EU-VO 524/2013 "
|
||||
"auf die OS-Plattform (https://ec.europa.eu/consumers/odr) "
|
||||
"verlinken — klickbarer Link, nicht nur Text. Zusaetzlich "
|
||||
"§36 VSBG: angeben, ob Sie an Verbraucher-"
|
||||
"Streitbeilegungsverfahren teilnehmen (oder nicht)."
|
||||
)
|
||||
else:
|
||||
check.skipped = True
|
||||
check.hint = "Nicht relevant (kein B2C Online-Shop)"
|
||||
|
||||
# Widerruf: Flag entire document as unnecessary for B2B
|
||||
if doc_type == "widerruf" and profile.business_type not in ("b2c", "unknown"):
|
||||
check.severity = "INFO"
|
||||
if not check.passed:
|
||||
check.hint = (
|
||||
"Als B2B-Unternehmen benoetigen Sie keine Widerrufsbelehrung "
|
||||
"(§355 BGB gilt nur fuer Verbrauchervertraege). "
|
||||
"Empfehlung: Entfernen Sie die Widerrufsbelehrung von "
|
||||
"Ihrer Website, da sie Verwirrung stiften kann."
|
||||
)
|
||||
|
||||
# Regulated profession: check for Kammer info
|
||||
if "kammer" in cid or "berufsordnung" in check.label.lower():
|
||||
if not profile.is_regulated_profession:
|
||||
check.skipped = True
|
||||
check.hint = "Nicht relevant (kein regulierter Beruf)"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _pad_results_with_missing(
|
||||
results: list,
|
||||
discovery_attempted: set[str] | None = None,
|
||||
) -> list:
|
||||
"""Ensure every canonical doc_type has an entry in the results list.
|
||||
|
||||
Doc_types the user did not submit AND auto-discovery did not find get
|
||||
a placeholder DocCheckResult. The error message distinguishes:
|
||||
- 'Auf der Website nicht gefunden' (discovery was attempted)
|
||||
- 'Nicht eingereicht' (no submitted URLs to crawl from)
|
||||
|
||||
Preserves the canonical ordering from _ALL_DOC_TYPES so the report
|
||||
layout is stable.
|
||||
"""
|
||||
from ..agent_doc_check_routes import DocCheckResult
|
||||
attempted = discovery_attempted or set()
|
||||
|
||||
by_type: dict[str, object] = {}
|
||||
for r in results:
|
||||
canon = "dse" if r.doc_type in ("datenschutz", "privacy") else r.doc_type
|
||||
by_type[canon] = r
|
||||
|
||||
ordered: list = []
|
||||
for dt in _ALL_DOC_TYPES:
|
||||
if dt in by_type:
|
||||
ordered.append(by_type[dt])
|
||||
continue
|
||||
if dt in attempted:
|
||||
msg = ("Auf der Website nicht gefunden — bitte URL des "
|
||||
"Dokuments manuell eintragen, falls vorhanden")
|
||||
else:
|
||||
msg = "Nicht eingereicht — Quelle nicht angegeben"
|
||||
ordered.append(DocCheckResult(
|
||||
label=_doc_type_label(dt),
|
||||
url="",
|
||||
doc_type=dt,
|
||||
word_count=0,
|
||||
completeness_pct=0,
|
||||
correctness_pct=0,
|
||||
checks=[],
|
||||
findings_count=0,
|
||||
error=msg,
|
||||
scenario="missing",
|
||||
))
|
||||
|
||||
extras = [r for r in results
|
||||
if (r.doc_type if r.doc_type not in ("datenschutz", "privacy") else "dse")
|
||||
not in _ALL_DOC_TYPES]
|
||||
ordered.extend(extras)
|
||||
return ordered
|
||||
|
||||
|
||||
def _result_to_dict(r) -> dict:
|
||||
"""Convert DocCheckResult to JSON-serializable dict."""
|
||||
fields = ("id", "label", "passed", "severity", "matched_text",
|
||||
"level", "parent", "skipped", "hint")
|
||||
return {
|
||||
"label": r.label, "url": r.url, "doc_type": r.doc_type,
|
||||
"word_count": r.word_count, "completeness_pct": r.completeness_pct,
|
||||
"correctness_pct": r.correctness_pct,
|
||||
"checks": [{f: getattr(c, f) for f in fields} for c in r.checks],
|
||||
"findings_count": r.findings_count, "error": r.error,
|
||||
"scenario": getattr(r, "scenario", ""),
|
||||
}
|
||||
|
||||
|
||||
def _build_profile_html(profile) -> str:
|
||||
from ..agent_doc_check_report import build_profile_html
|
||||
return build_profile_html(profile)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Thin orchestrator — runs the 6 phases of the compliance check.
|
||||
|
||||
The original `_run_compliance_check` was a 1620-line monolith. It is
|
||||
now decomposed into six phases (A=resolve, B=profile+check,
|
||||
C=banner+extract, D=report-build [D1 raw vendors, D2 finalize,
|
||||
D3-top/mid/bot blocks], E=email, F=persist), each in its own module.
|
||||
|
||||
State flows through a single mutable `dict` (see `_state.new_state`).
|
||||
This intentionally trades type safety for additive flexibility: the
|
||||
report-building phase routinely adds new optional keys for each new
|
||||
HTML block, and a typed dataclass would freeze the schema before the
|
||||
new blocks could land.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ._b1_wiring import run_b1
|
||||
from ._b3_wiring import run_b3
|
||||
from ._constants import _compliance_check_jobs
|
||||
from ._phase_a_resolve import run_phase_a
|
||||
from ._phase_b_profile_check import run_phase_b
|
||||
from ._phase_c_banner import run_phase_c
|
||||
from ._phase_d1_vendors_raw import run_phase_d1
|
||||
from ._phase_d2_vendors_finalize import run_phase_d2
|
||||
from ._phase_d3_blocks_bot import run_phase_d3_bot
|
||||
from ._phase_d3_blocks_mid import run_phase_d3_mid
|
||||
from ._phase_d3_blocks_top import run_phase_d3_top
|
||||
from ._phase_e_email import run_phase_e
|
||||
from ._phase_f_persist import run_phase_f
|
||||
from ._state import new_state
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_compliance_check(check_id: str, req) -> None:
|
||||
"""Background task: check all documents with business-profile context."""
|
||||
state = new_state(check_id, req)
|
||||
try:
|
||||
# Phase A: TDM gate + Step 1 (resolve / discover / split / dedup)
|
||||
continue_run = await run_phase_a(state)
|
||||
if not continue_run:
|
||||
return # TDM denied — job already marked skipped_tdm
|
||||
# Phase B: Step 2 (profile detect) + Step 3 (per-doc checks)
|
||||
await run_phase_b(state)
|
||||
# Phase C: Step 3b-d (banner + cross-check + TCF) + Step 4
|
||||
await run_phase_c(state)
|
||||
# Phase D-1/D-2: Step 5 vendor extraction + finalize
|
||||
await run_phase_d1(state)
|
||||
await run_phase_d2(state)
|
||||
# B1 + B3: cross-cutting checks that need the finalized vendor
|
||||
# list + DSI text. Render their own HTML blocks consumed by
|
||||
# phase D-3 bot's full_html composition.
|
||||
await run_b1(state)
|
||||
run_b3(state)
|
||||
# Phase D-3 top/mid/bot: Step 5 HTML blocks
|
||||
await run_phase_d3_top(state)
|
||||
await run_phase_d3_mid(state)
|
||||
await run_phase_d3_bot(state)
|
||||
# Phase E: Step 6 send mail (with A1 ZIP attachment)
|
||||
run_phase_e(state)
|
||||
# Phase F: Step 7 persist + audit log + unified findings
|
||||
run_phase_f(state)
|
||||
except Exception as e:
|
||||
logger.error("Compliance check %s failed: %s",
|
||||
check_id, e, exc_info=True)
|
||||
_compliance_check_jobs[check_id]["status"] = "failed"
|
||||
_compliance_check_jobs[check_id]["error"] = str(e)[:500]
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Phase A — TDM gate + text resolution + section split + dedup.
|
||||
|
||||
Covers (in the original `_run_compliance_check`):
|
||||
- TDM-reservation pre-check (§ 44b UrhG)
|
||||
- Step 1 Resolve texts (URL fetch / pasted text / auto-reclassify)
|
||||
- Step 1a Auto-discovery of missing canonical doc_types
|
||||
- Step 1b Section splitting (shared URL → multiple doc_types,
|
||||
DSI → Cookie/Social-Media auto-fill)
|
||||
- Step 1c Cross-document keyword search
|
||||
- P15 Dedup of doc_types referencing the same source document
|
||||
|
||||
Returns True to continue, False if the run was aborted (TDM denied).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ._constants import _compliance_check_jobs
|
||||
from ._discovery import _autodiscover_missing
|
||||
from ._fetch import _fetch_text
|
||||
from ._helpers import _update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_a(state: dict) -> bool:
|
||||
"""Run TDM gate + Step 1 + Step 1a-c + P15 dedup. Mutate state in place."""
|
||||
check_id = state["check_id"]
|
||||
req = state["req"]
|
||||
|
||||
# Reset anchor-locator cache per run (avoid cross-run leak)
|
||||
try:
|
||||
from compliance.services.doc_anchor_locator import reset_cache
|
||||
reset_cache()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# P7: TDM-Reservation-Check der Base-Domain (§ 44b UrhG).
|
||||
# Bei reserved/denied: Run sofort beenden, kein Crawl.
|
||||
try:
|
||||
from compliance.services.tdm_reservation_check import (
|
||||
check_tdm_reservation, is_crawl_allowed,
|
||||
)
|
||||
first_url = next(
|
||||
(d.url for d in req.documents if d.url), "",
|
||||
)
|
||||
if first_url:
|
||||
tdm = await check_tdm_reservation(first_url)
|
||||
_compliance_check_jobs[check_id]["tdm"] = tdm
|
||||
# P12: Bei tdm_override + Reason wird NICHT abgebrochen,
|
||||
# sondern nur dokumentiert. Override ohne Reason wird ignoriert.
|
||||
override_active = (
|
||||
req.tdm_override
|
||||
and len((req.tdm_override_reason or "").strip()) >= 10
|
||||
)
|
||||
if not is_crawl_allowed(tdm) and not override_active:
|
||||
_compliance_check_jobs[check_id]["status"] = "skipped_tdm"
|
||||
_compliance_check_jobs[check_id]["error"] = (
|
||||
f"TDM-Vorbehalt fuer {tdm.get('domain')} erkannt "
|
||||
f"(status={tdm.get('status')}) — Crawl nach § 44b "
|
||||
f"UrhG nicht zulaessig. Signals: "
|
||||
f"{[s.get('src') for s in tdm.get('signals', [])]}"
|
||||
)
|
||||
_compliance_check_jobs[check_id]["progress_pct"] = 100
|
||||
logger.info("TDM-skip check_id=%s domain=%s status=%s",
|
||||
check_id, tdm.get("domain"), tdm.get("status"))
|
||||
return False
|
||||
if override_active and not is_crawl_allowed(tdm):
|
||||
_compliance_check_jobs[check_id]["tdm_override"] = {
|
||||
"reason": req.tdm_override_reason.strip()[:500],
|
||||
"original_status": tdm.get("status"),
|
||||
}
|
||||
logger.warning(
|
||||
"TDM-Override aktiv: check_id=%s domain=%s "
|
||||
"status=%s reason=%r",
|
||||
check_id, tdm.get("domain"), tdm.get("status"),
|
||||
req.tdm_override_reason.strip()[:80],
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("TDM-check failed (proceeding): %s", e)
|
||||
|
||||
# Step 1: Resolve texts (fetch from URL if needed) — 0-30%
|
||||
_update(check_id, "Texte werden geladen...", 1)
|
||||
doc_texts: dict[str, str] = {}
|
||||
doc_entries: list[dict] = []
|
||||
|
||||
# Cache fetched URLs to detect duplicates
|
||||
url_text_cache: dict[str, str] = {}
|
||||
|
||||
n_docs = max(1, len(req.documents))
|
||||
# User-pasted-Tabellen-Vendors (kein LLM noetig) — werden weiter
|
||||
# unten in cmp_vendors gemerged.
|
||||
pasted_table_vendors: list[dict] = []
|
||||
for i, doc in enumerate(req.documents):
|
||||
pct = int(1 + (i / n_docs) * 29)
|
||||
_update(check_id, f"Texte laden {i+1}/{n_docs}: {doc.doc_type}...", pct)
|
||||
text = (doc.text or "").strip()
|
||||
input_source = "url"
|
||||
cmp_payloads: list[dict] = []
|
||||
if text:
|
||||
input_source = "text"
|
||||
if doc.url:
|
||||
input_source = "text+url" # User hat beide gefuellt
|
||||
logger.info(
|
||||
"doc_type=%s: User hat URL UND Text geliefert — "
|
||||
"Text gewinnt, URL wird als Quellen-Referenz behalten",
|
||||
doc.doc_type,
|
||||
)
|
||||
elif doc.url:
|
||||
url_key = doc.url.strip().rstrip("/").lower()
|
||||
if url_key in url_text_cache:
|
||||
text = url_text_cache[url_key]
|
||||
else:
|
||||
text, cmp_payloads = await _fetch_text(doc.url, doc_type=doc.doc_type)
|
||||
if text:
|
||||
url_text_cache[url_key] = text
|
||||
|
||||
# Auto-Reclassify-Check: wenn der user Text in das falsche
|
||||
# Doc-Type-Feld kopiert hat (z.B. Impressum-Text in DSE),
|
||||
# erkennen und ggf. umtaggen.
|
||||
actual_doc_type = doc.doc_type
|
||||
reclassify_hint: dict | None = None
|
||||
if input_source.startswith("text") and len(text) >= 500:
|
||||
try:
|
||||
from compliance.services.doc_type_classifier import (
|
||||
detect_mismatch,
|
||||
)
|
||||
reclassify_hint = detect_mismatch(doc.doc_type, text)
|
||||
if reclassify_hint and reclassify_hint["action"] == "reclassify":
|
||||
actual_doc_type = reclassify_hint["detected"]
|
||||
logger.info(
|
||||
"doc_type AUTO-RECLASSIFY: deklariert=%s "
|
||||
"erkannt=%s (score %d vs %d) — uebernehme erkannten Typ",
|
||||
doc.doc_type, actual_doc_type,
|
||||
reclassify_hint["detected_score"],
|
||||
reclassify_hint["declared_score"],
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("doc_type_classifier failed: %s", e)
|
||||
|
||||
# Cookie-Tabelle: wenn User Tabelle reinkopiert hat, deterministisch
|
||||
# parsen (kein LLM noetig) und Vendors gleich ableiten.
|
||||
if input_source.startswith("text") and actual_doc_type == "cookie":
|
||||
try:
|
||||
from compliance.services.cookies_table_parser import (
|
||||
parse_cookie_table,
|
||||
)
|
||||
tab_vendors = parse_cookie_table(text)
|
||||
if tab_vendors:
|
||||
pasted_table_vendors.extend(tab_vendors)
|
||||
logger.info(
|
||||
"Cookie-Tabelle erkannt im pasted Text — "
|
||||
"%d Vendors / %d Cookies deterministisch geparst",
|
||||
len(tab_vendors),
|
||||
sum(len(v.get("cookies", [])) for v in tab_vendors),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("cookies_table_parser failed: %s", e)
|
||||
|
||||
if text:
|
||||
doc_texts[actual_doc_type] = text
|
||||
doc_entries.append({
|
||||
"doc_type": actual_doc_type,
|
||||
"declared_doc_type": doc.doc_type,
|
||||
"url": doc.url,
|
||||
"text": text,
|
||||
"word_count": len(text.split()) if text else 0,
|
||||
"auto_discovered": False,
|
||||
"discovery_attempted": False,
|
||||
"cmp_payloads": cmp_payloads,
|
||||
"input_source": input_source,
|
||||
"reclassify_hint": reclassify_hint,
|
||||
})
|
||||
|
||||
# Step 1a-bis: AUTO-DISCOVERY
|
||||
await _autodiscover_missing(
|
||||
check_id, doc_entries, doc_texts, url_text_cache,
|
||||
)
|
||||
|
||||
# Step 1b: Section splitting — two cases:
|
||||
# 1. Same URL used for multiple doc_types → split by heading
|
||||
# 2. DSI text contains Cookie/Social-Media sections → auto-fill empty rows
|
||||
from compliance.services.section_splitter import (
|
||||
split_shared_texts, auto_fill_from_dsi, cross_search_documents,
|
||||
)
|
||||
split_shared_texts(doc_entries, url_text_cache)
|
||||
auto_fill_from_dsi(doc_entries)
|
||||
|
||||
# Step 1c: Cross-document search — find doc_types in wrong documents (30-35%)
|
||||
_update(check_id, "Dokumente werden uebergreifend durchsucht...", 32)
|
||||
placement_findings = cross_search_documents(doc_entries)
|
||||
|
||||
# Refresh doc_texts after all splitting/searching
|
||||
for entry in doc_entries:
|
||||
if entry.get("text"):
|
||||
doc_texts[entry["doc_type"]] = entry["text"]
|
||||
|
||||
# P15: Dedupe — wenn mehrere Doc-Types DASSELBE Dokument referenzieren
|
||||
# (z.B. Safetykon: User gibt /datenschutz fuer dse + cookie + widerruf),
|
||||
# behalten wir nur den primaeren Doc-Type. Andere: leeren + note.
|
||||
# Priorität: dse > impressum > cookie > widerruf > agb > nutzungsbedingungen
|
||||
_DOC_PRIORITY = ["dse", "impressum", "cookie", "widerruf", "agb",
|
||||
"nutzungsbedingungen", "social_media", "dsb"]
|
||||
seen_text_hash: dict[int, str] = {}
|
||||
for dt in _DOC_PRIORITY:
|
||||
entry = next((e for e in doc_entries if e.get("doc_type") == dt
|
||||
and e.get("text")), None)
|
||||
if not entry:
|
||||
continue
|
||||
text_hash = hash((entry.get("text") or "").strip()[:1000])
|
||||
if text_hash in seen_text_hash:
|
||||
primary = seen_text_hash[text_hash]
|
||||
logger.info(
|
||||
"P15 dedup: doc_type=%s referenziert dasselbe Dokument "
|
||||
"wie %s (URL=%s) -> als Duplikat markiert.",
|
||||
dt, primary, entry.get("url", "")[:60],
|
||||
)
|
||||
entry["text"] = ""
|
||||
entry["word_count"] = 0
|
||||
entry["url"] = ""
|
||||
entry["dup_of"] = primary
|
||||
doc_texts.pop(dt, None)
|
||||
else:
|
||||
seen_text_hash[text_hash] = dt
|
||||
|
||||
state["doc_texts"] = doc_texts
|
||||
state["doc_entries"] = doc_entries
|
||||
state["url_text_cache"] = url_text_cache
|
||||
state["pasted_table_vendors"] = pasted_table_vendors
|
||||
state["placement_findings"] = placement_findings
|
||||
return True
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Phase B — Business-profile detection + per-document checks.
|
||||
|
||||
Covers (in the original `_run_compliance_check`):
|
||||
- Step 2 Detect business profile (with optional homepage merge for
|
||||
P16 keywords)
|
||||
- Step 3 Run regex + MC + LLM checks on each submitted document
|
||||
(`_check_single`), applying skip rules + profile filter
|
||||
+ placement findings
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re as _re
|
||||
from dataclasses import asdict
|
||||
|
||||
import httpx
|
||||
|
||||
from ._helpers import (
|
||||
_apply_profile_filter,
|
||||
_doc_type_label,
|
||||
_get_skip_types,
|
||||
_update,
|
||||
)
|
||||
from ._single_check import _check_single
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_b(state: dict) -> None:
|
||||
"""Detect business profile + check each document. Mutates state in place."""
|
||||
check_id = state["check_id"]
|
||||
req = state["req"]
|
||||
doc_texts = state["doc_texts"]
|
||||
doc_entries = state["doc_entries"]
|
||||
placement_findings = state["placement_findings"]
|
||||
|
||||
# Step 2: Detect business profile (35-40%)
|
||||
from compliance.services.business_profiler import detect_business_profile
|
||||
_update(check_id, "Geschaeftsmodell wird erkannt...", 37)
|
||||
# P16: Homepage-Text mit fuer Profile-Detection (no_direct_sales
|
||||
# B2B-Indikatoren wie "CE-Zertifizierung" / "Schulungen" stehen oft
|
||||
# nur im Homepage-Menue, nicht im Pflichttext).
|
||||
profile_input = dict(doc_texts)
|
||||
try:
|
||||
base_url = ""
|
||||
for e in doc_entries:
|
||||
if e.get("url"):
|
||||
from urllib.parse import urlparse
|
||||
p = urlparse(e["url"])
|
||||
if p.scheme and p.netloc:
|
||||
base_url = f"{p.scheme}://{p.netloc}/"
|
||||
break
|
||||
if base_url:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=8.0, follow_redirects=True,
|
||||
headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) "
|
||||
"AppleWebKit/537.36 HeadlessChrome/120.0.0.0"},
|
||||
) as _hc:
|
||||
_hr = await _hc.get(base_url)
|
||||
if _hr.status_code == 200 and "text/html" in _hr.headers.get(
|
||||
"content-type", ""):
|
||||
_html = _hr.text[:60000]
|
||||
_html = _re.sub(r"<script[^>]*>.*?</script>", " ",
|
||||
_html, flags=_re.DOTALL | _re.IGNORECASE)
|
||||
_html = _re.sub(r"<style[^>]*>.*?</style>", " ",
|
||||
_html, flags=_re.DOTALL | _re.IGNORECASE)
|
||||
_html = _re.sub(r"<[^>]+>", " ", _html)
|
||||
_html = _re.sub(r"\s+", " ", _html).strip()
|
||||
if len(_html.split()) > 30:
|
||||
profile_input["__homepage"] = _html[:20000]
|
||||
logger.info("P16 homepage merged for profile: %d words",
|
||||
len(_html.split()))
|
||||
except Exception as e:
|
||||
logger.debug("homepage fetch for profile failed: %s", e)
|
||||
profile = await detect_business_profile(profile_input)
|
||||
profile_dict = asdict(profile)
|
||||
|
||||
# Step 3: Check each document
|
||||
from ..agent_doc_check_routes import CheckItem, DocCheckResult
|
||||
results: list[DocCheckResult] = []
|
||||
total_findings = 0
|
||||
use_agent_flag = req.use_agent or os.getenv(
|
||||
"COMPLIANCE_USE_AGENT", "false",
|
||||
).lower() == "true"
|
||||
|
||||
# Filter out doc_types that don't apply to this business profile
|
||||
skip_types = _get_skip_types(profile)
|
||||
|
||||
# Derive business_scope hints for the MC filter (O1 — Doc-type Scope-Flag).
|
||||
# MCs that explicitly require a feature (e.g. 'biometric_processing',
|
||||
# 'ai_decision_making', 'child_targeting') get dropped when the
|
||||
# detected profile doesn't declare it.
|
||||
business_scope: set[str] = set()
|
||||
for svc in (getattr(profile, "detected_services", []) or []):
|
||||
business_scope.add(str(svc).lower())
|
||||
if (getattr(profile, "business_type", "") or "").lower() == "b2c":
|
||||
business_scope.add("b2c")
|
||||
if getattr(profile, "has_online_shop", False):
|
||||
business_scope.add("ecommerce")
|
||||
if getattr(profile, "is_regulated_profession", False):
|
||||
business_scope.add("regulated_profession")
|
||||
|
||||
# Document checks: 40-80%
|
||||
n_entries = max(1, len(doc_entries))
|
||||
for i, entry in enumerate(doc_entries):
|
||||
text = entry["text"]
|
||||
doc_type = entry["doc_type"]
|
||||
label = _doc_type_label(doc_type)
|
||||
url = entry["url"]
|
||||
|
||||
if doc_type in skip_types:
|
||||
results.append(DocCheckResult(
|
||||
label=label, url=url, doc_type=doc_type,
|
||||
error=skip_types[doc_type],
|
||||
))
|
||||
continue
|
||||
|
||||
pct = int(40 + (i / n_entries) * 40)
|
||||
_update(check_id, f"Pruefen {i+1}/{n_entries}: {label}...", pct)
|
||||
|
||||
if not text or len(text) < 50:
|
||||
# P15: duplicate doc that was deduped against a primary doc
|
||||
if entry.get("dup_of"):
|
||||
results.append(DocCheckResult(
|
||||
label=label, url="", doc_type=doc_type,
|
||||
error=f"Nicht separat vorhanden — wird im Dokument "
|
||||
f"'{_doc_type_label(entry['dup_of'])}' "
|
||||
f"mit-geprueft.",
|
||||
))
|
||||
continue
|
||||
# P24: DSB-Kontakt ist Pflichtangabe in der DSE (Art. 13(1)(b)
|
||||
# DSGVO) — wenn kein separates DSB-Dokument vorliegt, ist das
|
||||
# KEIN Fehler. DSB-Pruefung passiert ohnehin in der DSE.
|
||||
if doc_type == "dsb" and not (entry.get("url") or "").strip():
|
||||
results.append(DocCheckResult(
|
||||
label=label, url="", doc_type=doc_type,
|
||||
error="Nicht separat vorhanden — DSB-Kontaktdaten "
|
||||
"werden in der Datenschutzerklaerung als "
|
||||
"Pflichtangabe nach Art. 13(1)(b) DSGVO geprueft.",
|
||||
))
|
||||
continue
|
||||
# Empty entry — either from auto-discovery padding (no URL
|
||||
# to fetch) or from a fetch that returned nothing. If there
|
||||
# was a URL we keep the error so the user knows the fetch
|
||||
# failed; otherwise let the padding step label it
|
||||
# 'Nicht eingereicht' / 'Auf der Website nicht gefunden'.
|
||||
if (entry.get("url") or "").strip():
|
||||
results.append(DocCheckResult(
|
||||
label=label, url=url, doc_type=doc_type,
|
||||
error="Kein Text vorhanden oder zu kurz",
|
||||
))
|
||||
continue
|
||||
|
||||
result = await _check_single(
|
||||
text, doc_type, label, url,
|
||||
entry["word_count"], use_agent_flag,
|
||||
business_scope=business_scope,
|
||||
business_profile={"no_direct_sales": getattr(profile, "no_direct_sales", False)},
|
||||
)
|
||||
|
||||
# Apply profile context filter
|
||||
result = _apply_profile_filter(result, profile, doc_type)
|
||||
|
||||
# Add placement findings — but only if the regex checks confirm
|
||||
# the text doesn't match. If completeness >= 50%, the text IS the
|
||||
# right doc_type despite missing cross-search keywords.
|
||||
if result.completeness_pct < 50:
|
||||
for pf in placement_findings:
|
||||
if pf.get("doc_type") == doc_type:
|
||||
result.checks.insert(0, CheckItem(**{
|
||||
k: v for k, v in pf.items() if k != "doc_type"
|
||||
}))
|
||||
|
||||
results.append(result)
|
||||
total_findings += result.findings_count
|
||||
|
||||
state["profile"] = profile
|
||||
state["profile_dict"] = profile_dict
|
||||
state["business_scope"] = business_scope
|
||||
state["results"] = results
|
||||
state["total_findings"] = total_findings
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Phase C — Banner scan + Cookie/DSE cross-check + TCF check + profile extract.
|
||||
|
||||
Covers (in the original `_run_compliance_check`):
|
||||
- Step 3b Cookie-banner scan via consent-tester /scan (homepage,
|
||||
3-phase consent test)
|
||||
- Step 3c Cross-check banner findings vs. cookie-policy text
|
||||
- Step 3d TCF vendor vs. DSI cross-check + VVT entries
|
||||
- Step 4 Extract profile hints from documents
|
||||
- Step 4b Determine scenario per document (skip / regenerate / fix /
|
||||
import)
|
||||
- Step 4c Pad missing canonical doc_types so the report always shows
|
||||
every checklist row
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from ._constants import CONSENT_TESTER_URL
|
||||
from ._helpers import _pad_results_with_missing, _update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_c(state: dict) -> None:
|
||||
"""Run banner scan + cross-checks + profile extraction. Mutates state."""
|
||||
check_id = state["check_id"]
|
||||
req = state["req"]
|
||||
doc_texts = state["doc_texts"]
|
||||
doc_entries = state["doc_entries"]
|
||||
results = state["results"]
|
||||
profile_dict = state["profile_dict"]
|
||||
|
||||
# Step 3b: Banner-Check (automatic, uses first URL or homepage)
|
||||
banner_result = None
|
||||
banner_url = req.documents[0].url if req.documents and req.documents[0].url else ""
|
||||
# Use the homepage (strip path) for banner check
|
||||
if banner_url:
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(banner_url)
|
||||
banner_url = f"{parsed.scheme}://{parsed.netloc}"
|
||||
if banner_url:
|
||||
_update(check_id, "Cookie-Banner wird geprueft...", 82)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=900.0) as client: # P50: +10min for vendor-detail-phase
|
||||
resp = await client.post(
|
||||
f"{CONSENT_TESTER_URL}/scan",
|
||||
json={"url": banner_url, "timeout_per_phase": 10},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
banner_result = resp.json()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Banner check failed: %s (%s)", e or "<empty>", type(e).__name__,
|
||||
)
|
||||
|
||||
# Step 3c: Cross-check Banner vs Cookie-Richtlinie (88-90%)
|
||||
if banner_result and "cookie" in doc_texts:
|
||||
from compliance.services.banner_cookie_cross_check import (
|
||||
cross_check_banner_vs_cookie,
|
||||
)
|
||||
from ..agent_doc_check_routes import CheckItem
|
||||
_update(check_id, "Banner vs. Cookie-Richtlinie abgleichen...", 89)
|
||||
cross_findings = cross_check_banner_vs_cookie(
|
||||
banner_result, doc_texts["cookie"],
|
||||
)
|
||||
if cross_findings:
|
||||
for r in results:
|
||||
if r.doc_type == "cookie":
|
||||
for cf in cross_findings:
|
||||
r.checks.append(CheckItem(**cf))
|
||||
l2 = [c for c in r.checks if c.level == 2 and not c.skipped]
|
||||
l2p = sum(1 for c in l2 if c.passed)
|
||||
r.correctness_pct = round(l2p / len(l2) * 100) if l2 else 0
|
||||
|
||||
# Step 3d: TCF Vendor cross-check against DSI
|
||||
tcf_vendors = banner_result.get("tcf_vendors", []) if banner_result else []
|
||||
vvt_entries: list[dict] = []
|
||||
if tcf_vendors and "dse" in doc_texts:
|
||||
_update(check_id, f"{len(tcf_vendors)} TCF-Verarbeiter vs. DSI abgleichen...", 91)
|
||||
from compliance.services.banner_cookie_cross_check import (
|
||||
cross_check_vendors_vs_dsi,
|
||||
)
|
||||
from compliance.services.vendor_vvt_mapper import map_vendors_to_vvt
|
||||
from ..agent_doc_check_routes import CheckItem
|
||||
vendor_findings = cross_check_vendors_vs_dsi(tcf_vendors, doc_texts["dse"])
|
||||
if vendor_findings:
|
||||
for r in results:
|
||||
if r.doc_type == "dse":
|
||||
for vf in vendor_findings:
|
||||
r.checks.append(CheckItem(**vf))
|
||||
vvt_entries = map_vendors_to_vvt(tcf_vendors)
|
||||
|
||||
# Step 4: Extract profile hints from documents (92-95%)
|
||||
_update(check_id, "Profil wird aus Dokumenten extrahiert...", 93)
|
||||
from compliance.services.profile_extractor import (
|
||||
extract_profile_from_documents,
|
||||
)
|
||||
extracted_profile = extract_profile_from_documents(doc_texts, profile_dict)
|
||||
|
||||
# Step 4b: Determine scenario per document
|
||||
for r in results:
|
||||
if r.error:
|
||||
r.scenario = "skip"
|
||||
elif r.completeness_pct < 30:
|
||||
r.scenario = "regenerate"
|
||||
elif r.completeness_pct < 95:
|
||||
r.scenario = "fix"
|
||||
else:
|
||||
r.scenario = "import"
|
||||
|
||||
# Step 4c: Always render all 8 canonical doc types. Missing types
|
||||
# are differentiated:
|
||||
# - Discovery was tried but found nothing -> 'Auf der Website
|
||||
# nicht gefunden' (suggest user provides URL manually)
|
||||
# - No submitted URLs at all -> 'Nicht eingereicht'
|
||||
attempted = {
|
||||
e["doc_type"] for e in doc_entries if e.get("discovery_attempted")
|
||||
}
|
||||
results = _pad_results_with_missing(results, discovery_attempted=attempted)
|
||||
|
||||
state["banner_result"] = banner_result
|
||||
state["banner_url"] = banner_url
|
||||
state["tcf_vendors"] = tcf_vendors
|
||||
state["vvt_entries"] = vvt_entries
|
||||
state["extracted_profile"] = extracted_profile
|
||||
state["results"] = results
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Phase D-1 — Vendor-extraction raw stages.
|
||||
|
||||
Covers (in the original Step 5 of `_run_compliance_check`):
|
||||
- Aggregate cmp_payloads from all doc_entries + banner_result (P30/P48)
|
||||
- Fallback: use DSE text when cookie was deduped (P17-D)
|
||||
- Extract structured vendor records from CMP payloads
|
||||
- LLM-cascade fallback when structured extract yields < 5 vendors (P52)
|
||||
- Phase-G vendor-details append (P57)
|
||||
- HTML-table DOM parse (Stage D)
|
||||
- Crawled cookie-table parse (Stage B)
|
||||
- Tesseract OCR over evidence slices (Stage C) — also captures the
|
||||
cookie_evidence_slices used by A1 e-mail attachment
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ._helpers import _company_name_from_url, _update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_d1(state: dict) -> None:
|
||||
"""Vendor-extract raw stages. Mutates state in place."""
|
||||
check_id = state["check_id"]
|
||||
doc_entries = state["doc_entries"]
|
||||
doc_texts = state["doc_texts"]
|
||||
banner_result = state["banner_result"]
|
||||
pasted_table_vendors = state["pasted_table_vendors"]
|
||||
|
||||
cmp_vendors: list[dict] = []
|
||||
cookie_payloads: list[dict] = []
|
||||
cookie_text = ""
|
||||
cookie_evidence_slices: list[dict] | None = None
|
||||
cookie_evidence_meta: dict | None = None
|
||||
|
||||
try:
|
||||
from compliance.services.vendor_extractor import (
|
||||
extract_vendors_from_payloads,
|
||||
)
|
||||
|
||||
# P30: aggregate cmp_payloads from ALL doc_entries — sites
|
||||
# like Mercedes load Usercentrics only on the homepage, so the
|
||||
# JSON gets captured during DSE/Impressum discovery, not in the
|
||||
# cookies.html fetch. Dedup by URL since the same payload is
|
||||
# captured on every page load.
|
||||
seen_cmp_urls: set[str] = set()
|
||||
for e in doc_entries:
|
||||
for p in (e.get("cmp_payloads") or []):
|
||||
p_url = p.get("url") or ""
|
||||
if p_url and p_url in seen_cmp_urls:
|
||||
continue
|
||||
seen_cmp_urls.add(p_url)
|
||||
cookie_payloads.append(p)
|
||||
if e.get("doc_type") == "cookie" and e.get("text"):
|
||||
cookie_text = e["text"]
|
||||
# P48: also pull cmp_payloads from the Banner-Scan (homepage 3-phase
|
||||
# consent test). Mercedes' Usercentrics-JSON is captured there even
|
||||
# when not in DSI-Discovery of static legal pages.
|
||||
if banner_result:
|
||||
for p in (banner_result.get("cmp_payloads") or []):
|
||||
p_url = p.get("url") or ""
|
||||
if p_url and p_url in seen_cmp_urls:
|
||||
continue
|
||||
seen_cmp_urls.add(p_url)
|
||||
cookie_payloads.append(p)
|
||||
if cookie_payloads:
|
||||
logger.info("P48: %d CMP-payloads available for vendor-extract "
|
||||
"(after Banner-Scan merge)", len(cookie_payloads))
|
||||
# P17-D: Fallback wenn cookie via P15 deduped wurde — nutze DSE-Text
|
||||
# sofern Cookie-Begriffe drin sind, damit LLM-Vendor-Extract trotzdem
|
||||
# greifen kann.
|
||||
if not cookie_text and not cookie_payloads:
|
||||
dse_t = doc_texts.get("dse", "")
|
||||
if dse_t and any(w in dse_t.lower() for w in
|
||||
("cookie", "tracking", "google analytics", "consent")):
|
||||
cookie_text = dse_t
|
||||
logger.info("P17-D: vendor-extract Fallback auf DSE (Cookie deduped)")
|
||||
owner_name = _company_name_from_url(doc_entries) or ""
|
||||
if cookie_payloads:
|
||||
cmp_vendors = extract_vendors_from_payloads(
|
||||
cookie_payloads, owner_name=owner_name,
|
||||
)
|
||||
|
||||
# P52: LLM-Fallback nicht nur wenn 0 Vendors, sondern auch wenn die
|
||||
# strukturierten Quellen < 5 Vendors lieferten und der Cookie-Text
|
||||
# substantiell ist.
|
||||
if (len(cmp_vendors) < 5
|
||||
and cookie_text and len(cookie_text.split()) >= 500):
|
||||
from compliance.services.vendor_llm_extractor import (
|
||||
extract_vendors_via_llm,
|
||||
)
|
||||
from compliance.services.vendor_classifier import classify
|
||||
_update(check_id, "Vendor-Liste per LLM extrahieren...", 94)
|
||||
llm_vendors = await extract_vendors_via_llm(cookie_text)
|
||||
existing_names = {(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors}
|
||||
added_llm = 0
|
||||
for v in llm_vendors:
|
||||
nm = (v.get("name") or "").strip()
|
||||
if not nm or nm.lower() in existing_names:
|
||||
continue
|
||||
v["recipient_type"] = classify(
|
||||
vendor_name=nm,
|
||||
category=v.get("category", ""),
|
||||
owner_name=owner_name,
|
||||
)
|
||||
v.setdefault("source", "llm_cascade")
|
||||
cmp_vendors.append(v)
|
||||
existing_names.add(nm.lower())
|
||||
added_llm += 1
|
||||
if added_llm:
|
||||
logger.info("P52 LLM-Cascade: +%d Vendors (total: %d)",
|
||||
added_llm, len(cmp_vendors))
|
||||
|
||||
# P57: Phase G vendor_details als zusätzliche Vendor-Quelle.
|
||||
if banner_result:
|
||||
vd_list = banner_result.get("vendor_details") or []
|
||||
vd_list = [v for v in vd_list if v.get("name") != "__TDM_OPTOUT__"]
|
||||
existing_names = {(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors}
|
||||
added = 0
|
||||
for d in vd_list:
|
||||
n = (d.get("name") or "").strip()
|
||||
if not n or n.lower() in existing_names:
|
||||
continue
|
||||
if n.lower() in ("technisch erforderlich", "analyse und statistik",
|
||||
"marketing", "alles auswählen",
|
||||
"alles auswaehlen"):
|
||||
continue
|
||||
from compliance.services.vendor_classifier import classify
|
||||
cmp_vendors.append({
|
||||
"name": n,
|
||||
"country": "",
|
||||
"purpose": d.get("description", "")[:500],
|
||||
"category": "",
|
||||
"opt_out_url": d.get("opt_out_url", ""),
|
||||
"privacy_policy_url": d.get("privacy_url", ""),
|
||||
"persistence": d.get("retention", ""),
|
||||
"cookies": d.get("cookies", []),
|
||||
"processing_company": d.get("processing_company", ""),
|
||||
"address": d.get("address", ""),
|
||||
"purposes": d.get("purposes", []),
|
||||
"technologies": d.get("technologies", []),
|
||||
"recipient_type": classify(
|
||||
vendor_name=n, category="", owner_name=owner_name,
|
||||
),
|
||||
})
|
||||
existing_names.add(n.lower())
|
||||
added += 1
|
||||
if added:
|
||||
logger.info("P57: added %d new vendors from Phase G (total: %d)",
|
||||
added, len(cmp_vendors))
|
||||
|
||||
# D — HTML-Tabellen aus DOM
|
||||
for pl in (cookie_payloads or []):
|
||||
if pl.get("kind") != "html_table":
|
||||
continue
|
||||
rows = pl.get("rows") or []
|
||||
if len(rows) < 3:
|
||||
continue
|
||||
try:
|
||||
from compliance.services.cookies_table_parser import (
|
||||
parse_cookie_table as _parse_ct_d,
|
||||
)
|
||||
table_text = "\n".join(rows)
|
||||
d_vendors = _parse_ct_d(table_text)
|
||||
if d_vendors:
|
||||
existing_d = {(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors}
|
||||
added_d = 0
|
||||
for v in d_vendors:
|
||||
nm = (v.get("name") or "").strip()
|
||||
if not nm or nm.lower() in existing_d:
|
||||
continue
|
||||
v.setdefault("source", "html_table_dom")
|
||||
cmp_vendors.append(v)
|
||||
existing_d.add(nm.lower())
|
||||
added_d += 1
|
||||
if added_d:
|
||||
logger.info("D HTML-Table-DOM-Parse: +%d Vendors aus "
|
||||
"%d-Zeilen-Tabelle (total: %d)",
|
||||
added_d, len(rows), len(cmp_vendors))
|
||||
except Exception as e:
|
||||
logger.warning("html_table parse failed: %s", e)
|
||||
|
||||
# B — cookies_table_parser auch auf gecrawltem Cookie-Text
|
||||
if cookie_text and len(cookie_text) >= 500:
|
||||
try:
|
||||
from compliance.services.cookies_table_parser import (
|
||||
parse_cookie_table as _parse_ct,
|
||||
parse_flat_cookie_text as _parse_flat,
|
||||
)
|
||||
crawled_table_vendors = _parse_ct(cookie_text)
|
||||
if not crawled_table_vendors:
|
||||
crawled_table_vendors = _parse_flat(cookie_text)
|
||||
if crawled_table_vendors:
|
||||
existing = {(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors}
|
||||
added_c = 0
|
||||
for v in crawled_table_vendors:
|
||||
nm = (v.get("name") or "").strip()
|
||||
if not nm or nm.lower() in existing:
|
||||
continue
|
||||
v.setdefault("source", "table_crawled")
|
||||
cmp_vendors.append(v)
|
||||
existing.add(nm.lower())
|
||||
added_c += 1
|
||||
if added_c:
|
||||
logger.info("B Crawled-Tabellen-Parse: +%d Vendors "
|
||||
"(total: %d)", added_c, len(cmp_vendors))
|
||||
except Exception as e:
|
||||
logger.warning("crawled-table-parse failed: %s", e)
|
||||
|
||||
# C — Screenshot + Tesseract-OCR (auch Quelle für A1 ZIP-Anhang)
|
||||
cookie_url_for_shot = ""
|
||||
for _e in doc_entries:
|
||||
if _e.get("doc_type") == "cookie" and _e.get("url"):
|
||||
cookie_url_for_shot = _e["url"]; break
|
||||
if cookie_url_for_shot:
|
||||
try:
|
||||
from compliance.services.cookie_screenshot_ocr import (
|
||||
capture_cookie_evidence_slices,
|
||||
cookies_to_vendor_records,
|
||||
ocr_slices_extract_cookies,
|
||||
)
|
||||
from compliance.services.cookies_table_parser import (
|
||||
_guess_vendor as _gv,
|
||||
)
|
||||
_update(check_id,
|
||||
"Cookie-Richtlinie wird fotografiert "
|
||||
"(lueckenlose Beweiskette)...", 92)
|
||||
ev = await capture_cookie_evidence_slices(
|
||||
cookie_url_for_shot, check_id=check_id,
|
||||
viewport_h=1024, overlap_px=200, max_slices=40,
|
||||
)
|
||||
if ev.get("slices"):
|
||||
cookie_evidence_slices = ev["slices"]
|
||||
cookie_evidence_meta = {
|
||||
"total_height_px": ev.get("total_height_px"),
|
||||
"width_px": ev.get("width_px"),
|
||||
"accepted_banner": ev.get("accepted_banner"),
|
||||
"expanded": ev.get("expanded"),
|
||||
"url": ev.get("url"),
|
||||
"slice_count": len(ev["slices"]),
|
||||
}
|
||||
_update(check_id, "Tesseract OCR über alle Slices...", 93)
|
||||
ocr_cookies, ocr_stats = ocr_slices_extract_cookies(
|
||||
ev["slices"],
|
||||
)
|
||||
if ocr_cookies:
|
||||
ocr_vendors = cookies_to_vendor_records(
|
||||
ocr_cookies, guess_vendor_fn=_gv,
|
||||
)
|
||||
existing = {(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors}
|
||||
added_v = 0
|
||||
for v in ocr_vendors:
|
||||
nm = (v.get("name") or "").strip()
|
||||
if not nm:
|
||||
continue
|
||||
if nm.lower() in existing:
|
||||
for ex in cmp_vendors:
|
||||
if (ex.get("name") or "").strip().lower() == nm.lower():
|
||||
ex_names = {
|
||||
(c.get("name") or "").lower()
|
||||
for c in (ex.get("cookies") or [])
|
||||
}
|
||||
for c in (v.get("cookies") or []):
|
||||
if c["name"].lower() not in ex_names:
|
||||
ex.setdefault("cookies", []).append(c)
|
||||
ex_names.add(c["name"].lower())
|
||||
cur_src = ex.get("source", "")
|
||||
if "tesseract_ocr" not in cur_src:
|
||||
ex["source"] = (cur_src + ";tesseract_ocr").strip(";")
|
||||
break
|
||||
continue
|
||||
cmp_vendors.append(v)
|
||||
existing.add(nm.lower())
|
||||
added_v += 1
|
||||
logger.info(
|
||||
"C Tesseract-OCR: +%d Vendors / %d Cookies "
|
||||
"(über %d Slices, total: %d)",
|
||||
added_v, len(ocr_cookies),
|
||||
ocr_stats.get("slices", 0), len(cmp_vendors),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Tesseract-OCR pipeline failed: %s (%s)",
|
||||
str(e) or "(no msg)", type(e).__name__)
|
||||
|
||||
# User-pasted Cookie-Tabelle (deterministisch, kein LLM):
|
||||
# die hat IMMER Vorrang weil 100% genau.
|
||||
if pasted_table_vendors:
|
||||
existing = {(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors}
|
||||
added_p = 0
|
||||
for v in pasted_table_vendors:
|
||||
nm = (v.get("name") or "").strip()
|
||||
if not nm or nm.lower() in existing:
|
||||
continue
|
||||
cmp_vendors.append(v)
|
||||
existing.add(nm.lower())
|
||||
added_p += 1
|
||||
if added_p:
|
||||
logger.info("Pasted-Tabellen-Merge: +%d Vendors (total: %d)",
|
||||
added_p, len(cmp_vendors))
|
||||
except Exception as e:
|
||||
logger.warning("VVT vendor extraction skipped: %s", e)
|
||||
|
||||
state["cmp_vendors"] = cmp_vendors
|
||||
state["cookie_payloads"] = cookie_payloads
|
||||
state["cookie_text"] = cookie_text
|
||||
state["cookie_evidence_slices"] = cookie_evidence_slices
|
||||
state["cookie_evidence_meta"] = cookie_evidence_meta
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Phase D-2 — Vendor finalize: enrich + normalize + library fallback.
|
||||
|
||||
Covers (in the original Step 5 of `_run_compliance_check`):
|
||||
- Cookie-Library-Fallback (P52 Lite) — when < 20 vendors but many
|
||||
after-accept cookies, resolve via library
|
||||
- Vendor-Normalizer (Google-Familie dedup, garbage filter)
|
||||
- Detail-modal enrichment from Phase G (P50) + TDM-opt-out sentinel
|
||||
- Cookie-Behavior-Validator (P59b) — 3-Tier severity findings
|
||||
- Implicit cookies detection (P61) — GTM brings GA/GCL/DoubleClick
|
||||
- validate_vendor_urls + score_vendors + cookie-function classify
|
||||
- Vendor-Redundanz (O4) + EU-Alternativen + Cost/Savings
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_d2(state: dict) -> None:
|
||||
"""Vendor finalize stages + redundancy. Mutates state in place."""
|
||||
cmp_vendors = state["cmp_vendors"]
|
||||
cookie_text = state.get("cookie_text", "")
|
||||
banner_result = state["banner_result"]
|
||||
banner_url = state["banner_url"]
|
||||
profile = state["profile"]
|
||||
business_scope = state["business_scope"]
|
||||
|
||||
tdm_opt_out_notice = ""
|
||||
cookie_behavior_findings: list[dict] = []
|
||||
redundancy_report = None
|
||||
|
||||
try:
|
||||
from compliance.services.cookie_link_validator import (
|
||||
score_vendors, validate_vendor_urls,
|
||||
)
|
||||
|
||||
# Cookie-Library-Fallback (P52 Lite): wenn weiterhin wenige
|
||||
# Vendors aber viele after_accept-Cookies, aus Library auflösen.
|
||||
# VW-Lehre: 6 LLM-Grob-Vendors reichen NICHT — die Library
|
||||
# holt 30+ weitere aus den Cookie-Namen + Cookie-Doc-Pattern.
|
||||
# Schwelle: immer probieren wenn < 20 Vendors.
|
||||
if banner_result and len(cmp_vendors) < 20:
|
||||
try:
|
||||
from compliance.services.cookie_to_vendor_fallback import (
|
||||
fallback_vendors_for_run,
|
||||
)
|
||||
from database import SessionLocal as _SLfb
|
||||
_fb_db = _SLfb()
|
||||
try:
|
||||
extra = fallback_vendors_for_run(
|
||||
_fb_db, banner_result, len(cmp_vendors),
|
||||
cookie_doc_text=cookie_text,
|
||||
)
|
||||
if extra:
|
||||
existing_names = {(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors}
|
||||
for v in extra:
|
||||
if v["name"].lower() in existing_names:
|
||||
continue
|
||||
cmp_vendors.append(v)
|
||||
logger.info(
|
||||
"Cookie-Library-Fallback: cmp_vendors %d -> %d",
|
||||
len(cmp_vendors) - len(extra), len(cmp_vendors),
|
||||
)
|
||||
finally:
|
||||
_fb_db.close()
|
||||
except Exception as e:
|
||||
logger.warning("Cookie-Library-Fallback skipped: %s", e)
|
||||
|
||||
# Vendor-Normalizer: Dedup (Google-Familie etc) + Garbage-Filter
|
||||
try:
|
||||
from compliance.services.vendor_normalizer import (
|
||||
normalize_vendors as _norm_v,
|
||||
)
|
||||
cmp_vendors = _norm_v(cmp_vendors)
|
||||
except Exception as e:
|
||||
logger.warning("vendor_normalizer skipped: %s", e)
|
||||
|
||||
# P50: enrich vendors with per-vendor detail-modal-extracts
|
||||
if cmp_vendors and banner_result:
|
||||
vendor_details = banner_result.get("vendor_details") or []
|
||||
# P50f: filter out TDM-opt-out sentinel
|
||||
tdm_sentinel = next((v for v in vendor_details
|
||||
if v.get("name") == "__TDM_OPTOUT__"), None)
|
||||
if tdm_sentinel:
|
||||
tdm_opt_out_notice = tdm_sentinel.get("description", "")
|
||||
logger.info("P50f: TDM opt-out — skipped detail-enrichment for vendors")
|
||||
vendor_details = [v for v in vendor_details
|
||||
if v.get("name") != "__TDM_OPTOUT__"]
|
||||
if vendor_details:
|
||||
details_by_name = {}
|
||||
for d in vendor_details:
|
||||
n = (d.get("name") or "").strip().lower()
|
||||
if n:
|
||||
details_by_name[n] = d
|
||||
enriched = 0
|
||||
for v in cmp_vendors:
|
||||
key = (v.get("name") or "").strip().lower()
|
||||
d = details_by_name.get(key)
|
||||
if not d:
|
||||
for dn, dv in details_by_name.items():
|
||||
if key in dn or dn in key:
|
||||
d = dv
|
||||
break
|
||||
if not d:
|
||||
continue
|
||||
if not v.get("country") and (d.get("processing_company") or d.get("address")):
|
||||
addr = d.get("address", "")
|
||||
if re.search(r"\b(deutschland|germany|berlin|m(?:ue|ü)nchen|hamburg|stuttgart)\b", addr, re.I):
|
||||
v["country"] = "DE"
|
||||
elif re.search(r"\bireland|irland|dublin\b", addr, re.I):
|
||||
v["country"] = "IE"
|
||||
elif re.search(r"\busa|united states|california|new york|delaware\b", addr, re.I):
|
||||
v["country"] = "US"
|
||||
if not v.get("purpose"):
|
||||
v["purpose"] = d.get("description", "")[:500]
|
||||
if not v.get("opt_out_url"):
|
||||
v["opt_out_url"] = d.get("opt_out_url", "")
|
||||
if not v.get("privacy_policy_url"):
|
||||
v["privacy_policy_url"] = d.get("privacy_url", "")
|
||||
if not v.get("cookies"):
|
||||
v["cookies"] = d.get("cookies", [])
|
||||
v["purposes"] = d.get("purposes", [])
|
||||
v["technologies"] = d.get("technologies", [])
|
||||
if not v.get("persistence"):
|
||||
v["persistence"] = d.get("retention", "")
|
||||
v["processing_company"] = d.get("processing_company", "")
|
||||
v["address"] = d.get("address", "")
|
||||
enriched += 1
|
||||
logger.info("P50: enriched %d/%d vendors with detail-modal data",
|
||||
enriched, len(cmp_vendors))
|
||||
|
||||
# P59b: Cookie-Behavior-Validator
|
||||
if banner_result:
|
||||
cookies_detailed = banner_result.get("cookies_detailed") or []
|
||||
if cookies_detailed:
|
||||
cb_session = None
|
||||
try:
|
||||
from database import SessionLocal
|
||||
from compliance.services.cookie_behavior_validator import (
|
||||
validate_cookie_behavior,
|
||||
)
|
||||
from urllib.parse import urlparse
|
||||
fp_domain = ""
|
||||
if banner_url:
|
||||
fp_domain = urlparse(banner_url).netloc.replace("www.", "")
|
||||
cb_session = SessionLocal()
|
||||
cookie_behavior_findings = validate_cookie_behavior(
|
||||
cb_session, cookies_detailed,
|
||||
network_requests=[], # TODO Layer B in P59d
|
||||
first_party_domain=fp_domain,
|
||||
)
|
||||
if cookie_behavior_findings:
|
||||
sevs = {f["severity"] for f in cookie_behavior_findings}
|
||||
logger.info(
|
||||
"P59b: Cookie-Behavior-Check %d findings (severities: %s) "
|
||||
"ueber %d Cookies",
|
||||
len(cookie_behavior_findings),
|
||||
sorted(sevs), len(cookies_detailed),
|
||||
)
|
||||
banner_result["cookie_behavior_findings"] = (
|
||||
cookie_behavior_findings
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"P59b: Cookie-Behavior-Check 0 findings ueber %d Cookies "
|
||||
"(library miss / clean)", len(cookies_detailed),
|
||||
)
|
||||
except Exception as cb_err:
|
||||
logger.warning("P59b Cookie-Behavior-Check failed: %s", cb_err)
|
||||
finally:
|
||||
if cb_session is not None:
|
||||
try:
|
||||
cb_session.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# P61: "Untergeschobene Cookies"
|
||||
if banner_result and cmp_vendors:
|
||||
try:
|
||||
from compliance.services.vendor_package_cookies import (
|
||||
detect_implicit_cookies,
|
||||
)
|
||||
declared = [v.get("name", "") for v in cmp_vendors if v.get("name")]
|
||||
actual_cookies: list[str] = []
|
||||
for phase_data in (banner_result.get("phases") or {}).values():
|
||||
if isinstance(phase_data, dict):
|
||||
for ck in (phase_data.get("cookies") or []):
|
||||
if isinstance(ck, dict) and ck.get("name"):
|
||||
actual_cookies.append(ck["name"])
|
||||
implicit_findings = detect_implicit_cookies(
|
||||
declared, actual_cookies_set=actual_cookies or None,
|
||||
)
|
||||
if implicit_findings:
|
||||
banner_result["implicit_vendor_findings"] = implicit_findings
|
||||
logger.info(
|
||||
"P61: %d implicit vendor-package items detected "
|
||||
"(%d cookies + %d vendors)",
|
||||
len(implicit_findings),
|
||||
sum(1 for f in implicit_findings if f["implicit"]["type"] == "cookie"),
|
||||
sum(1 for f in implicit_findings if f["implicit"]["type"] == "vendor"),
|
||||
)
|
||||
except Exception as p61_err:
|
||||
logger.warning("P61 implicit-vendor detection failed: %s", p61_err)
|
||||
|
||||
if cmp_vendors:
|
||||
logger.info("VVT: %d vendors extracted, validating links",
|
||||
len(cmp_vendors))
|
||||
cmp_vendors = await validate_vendor_urls(cmp_vendors)
|
||||
cmp_vendors = score_vendors(cmp_vendors)
|
||||
try:
|
||||
from compliance.services.cookie_function_classifier import (
|
||||
annotate_vendor_cookies,
|
||||
)
|
||||
cmp_vendors = [annotate_vendor_cookies(v) for v in cmp_vendors]
|
||||
except Exception as e:
|
||||
logger.warning("Cookie function classification skipped: %s", e)
|
||||
except Exception as e:
|
||||
logger.warning("VVT vendor finalize skipped: %s", e)
|
||||
|
||||
# Vendor-Redundanz + EU-Alternativen + Cost/Savings (O4)
|
||||
try:
|
||||
from compliance.services.vendor_cost_estimator import infer_company_tier
|
||||
from compliance.services.vendor_redundancy import (
|
||||
analyze as analyze_redundancy,
|
||||
)
|
||||
if cmp_vendors:
|
||||
bp_dict = {
|
||||
"type": getattr(profile, "business_type", ""),
|
||||
"features": list(business_scope),
|
||||
}
|
||||
ctier = infer_company_tier(bp_dict)
|
||||
redundancy_report = analyze_redundancy(cmp_vendors, company_tier=ctier)
|
||||
logger.info(
|
||||
"Redundanz: %d Kategorien mit Mehrfach-Anbietern, "
|
||||
"Spar-Schaetzung %s pro Jahr (company_tier=%s)",
|
||||
redundancy_report["summary"]["redundancy_count"],
|
||||
redundancy_report["summary"]["estimated_saving_pct"],
|
||||
ctier,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Vendor redundancy analysis skipped: %s", e)
|
||||
|
||||
state["cmp_vendors"] = cmp_vendors
|
||||
state["tdm_opt_out_notice"] = tdm_opt_out_notice
|
||||
state["cookie_behavior_findings"] = cookie_behavior_findings
|
||||
state["redundancy_report"] = redundancy_report
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Phase D-3-Bot — Bottom HTML blocks + final composition.
|
||||
|
||||
Covers (in the original Step 5):
|
||||
- P71 JC-vs-AVV Entscheidungsbaum (only when DSE ambig)
|
||||
- P6/P53/P55 Branchen-Kontext + Site-History
|
||||
- P106 Internal-Checks-Block
|
||||
- P85 Banner-Screenshot
|
||||
- A Audit-Quality-Checks (Banner-Detect-Failure, vendor-extract dünn)
|
||||
- P82 GF-1-Pager
|
||||
- Doc-Input-Warnings (User text in falsches Feld gepastet)
|
||||
- P86 Branchen-Benchmark
|
||||
- P84 Diff-Mode (since-last-run delta)
|
||||
- Final HTML composition
|
||||
|
||||
NOTE: in the original code `audit_quality_findings` was used by
|
||||
build_gf_one_pager_html BEFORE it was initialised — a silent
|
||||
UnboundLocalError caught by the surrounding try/except, so the
|
||||
gf_one_pager block effectively never rendered. Here we run
|
||||
audit-quality FIRST so the data is actually available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_d3_bot(state: dict) -> None:
|
||||
"""Bottom blocks + assemble full_html. Mutates state in place."""
|
||||
check_id = state["check_id"]
|
||||
req = state["req"]
|
||||
doc_entries = state["doc_entries"]
|
||||
doc_texts = state["doc_texts"]
|
||||
banner_result = state["banner_result"]
|
||||
cmp_vendors = state["cmp_vendors"]
|
||||
mc_split = state["mc_split"]
|
||||
scorecard = state["scorecard"]
|
||||
prev_scorecard = state.get("prev_scorecard")
|
||||
mismatches = state.get("mismatches") or []
|
||||
site_name_for_exec = state.get("site_name_for_exec", "")
|
||||
domain_for_exec = state.get("domain_for_exec")
|
||||
html_blocks = state["html_blocks"]
|
||||
|
||||
# P71: JC-vs-AVV Entscheidungsbaum
|
||||
jc_decision_html = ""
|
||||
try:
|
||||
from compliance.services.jc_avv_decision import (
|
||||
build_jc_avv_decision_html,
|
||||
)
|
||||
jc_decision_html = build_jc_avv_decision_html(doc_texts.get("dse"))
|
||||
except Exception as e:
|
||||
logger.warning("P71 jc_avv_decision skipped: %s", e)
|
||||
|
||||
# P6/P53/P55 — Branchen-Kontext + Site-History
|
||||
industry_ctx_html = ""
|
||||
try:
|
||||
from compliance.services.industry_library import (
|
||||
build_industry_context_block_html, load_site_profile,
|
||||
)
|
||||
from database import SessionLocal as _SLib
|
||||
_ind_db = _SLib()
|
||||
try:
|
||||
ind = (req.scan_context or {}).get("industry") if req.scan_context else None
|
||||
site_prof = load_site_profile(_ind_db, domain_for_exec or "")
|
||||
industry_ctx_html = build_industry_context_block_html(ind, site_prof)
|
||||
finally:
|
||||
_ind_db.close()
|
||||
except Exception as e:
|
||||
logger.warning("industry context skipped: %s", e)
|
||||
|
||||
# P106 — Internal-Checks-Block
|
||||
internal_checks_html = ""
|
||||
try:
|
||||
from compliance.services.mc_audit_type import (
|
||||
build_internal_checks_block_html,
|
||||
)
|
||||
ic = (mc_split or {}).get("internal_checks") or []
|
||||
if ic:
|
||||
internal_checks_html = build_internal_checks_block_html(ic)
|
||||
logger.info("P106: %d interne Checks (statt FAIL) im Block",
|
||||
len(ic))
|
||||
except Exception as e:
|
||||
logger.warning("P106 internal_checks_html skipped: %s", e)
|
||||
|
||||
# P85 — Banner-Screenshot
|
||||
banner_shot_html = ""
|
||||
try:
|
||||
from compliance.services.banner_screenshot_block import (
|
||||
build_banner_screenshot_html,
|
||||
)
|
||||
banner_shot_html = build_banner_screenshot_html(banner_result)
|
||||
except Exception as e:
|
||||
logger.warning("P85 banner-screenshot skipped: %s", e)
|
||||
|
||||
# A — Audit-Quality-Checks (run BEFORE gf_one_pager so the data is
|
||||
# available — original code had this inverted, causing
|
||||
# UnboundLocalError silently caught).
|
||||
audit_quality_html = ""
|
||||
audit_quality_findings: list[dict] = []
|
||||
try:
|
||||
from compliance.services.audit_quality_checks import (
|
||||
build_audit_quality_block_html, run_all as run_audit_quality,
|
||||
)
|
||||
cookie_text_for_aq = doc_texts.get("cookie") or ""
|
||||
audit_quality_findings = run_audit_quality(
|
||||
banner_result, cookie_text_for_aq, cmp_vendors, doc_entries,
|
||||
)
|
||||
if audit_quality_findings:
|
||||
audit_quality_html = build_audit_quality_block_html(audit_quality_findings)
|
||||
logger.info("audit-quality: %d Vorbehalte erkannt",
|
||||
len(audit_quality_findings))
|
||||
except Exception as e:
|
||||
logger.warning("audit-quality-checks failed: %s", e)
|
||||
|
||||
# P82: GF-1-Pager (now has the audit_quality_findings filled)
|
||||
gf_one_pager_html = ""
|
||||
try:
|
||||
from compliance.services.gf_one_pager import build_gf_one_pager_html
|
||||
gf_one_pager_html = build_gf_one_pager_html(
|
||||
site_name=site_name_for_exec,
|
||||
scorecard=scorecard,
|
||||
previous_scorecard=prev_scorecard,
|
||||
banner_result=banner_result,
|
||||
library_mismatch_findings=mismatches,
|
||||
scan_context=req.scan_context,
|
||||
audit_quality_findings=audit_quality_findings,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("P82 GF-1-pager skipped: %s", e)
|
||||
|
||||
# Doc-Input-Warnings — wenn User Text ins falsche Feld gepastet hat
|
||||
input_warn_html = ""
|
||||
try:
|
||||
from compliance.services.doc_input_warnings import (
|
||||
build_warnings_block_html, collect_warnings,
|
||||
)
|
||||
warns = collect_warnings(doc_entries)
|
||||
if warns:
|
||||
input_warn_html = build_warnings_block_html(warns)
|
||||
logger.info("doc-input-warnings: %d Mismatches gefunden", len(warns))
|
||||
except Exception as e:
|
||||
logger.warning("doc-input-warnings skipped: %s", e)
|
||||
|
||||
# P86: Branchen-Benchmark
|
||||
bench_html = ""
|
||||
try:
|
||||
from compliance.services.industry_benchmark import (
|
||||
_extract_score, build_benchmark_html, compute_benchmark,
|
||||
)
|
||||
from database import SessionLocal as _SLb
|
||||
industry = (req.scan_context or {}).get("industry") if req.scan_context else None
|
||||
curr_score = _extract_score(banner_result)
|
||||
if industry and curr_score is not None:
|
||||
_b_db = _SLb()
|
||||
try:
|
||||
bench = compute_benchmark(
|
||||
_b_db, industry, curr_score, check_id,
|
||||
)
|
||||
if bench:
|
||||
bench_html = build_benchmark_html(bench)
|
||||
finally:
|
||||
_b_db.close()
|
||||
except Exception as e:
|
||||
logger.warning("P86 industry-benchmark skipped: %s", e)
|
||||
|
||||
# P84: Diff-Mode
|
||||
diff_html = ""
|
||||
try:
|
||||
from compliance.services.run_diff import (
|
||||
build_diff_block_html, compute_diff,
|
||||
)
|
||||
from database import SessionLocal as _SL
|
||||
_diff_db = _SL()
|
||||
try:
|
||||
diff = compute_diff(
|
||||
_diff_db, check_id, domain_for_exec or "",
|
||||
banner_result, scorecard,
|
||||
)
|
||||
if diff:
|
||||
diff_html = build_diff_block_html(diff)
|
||||
finally:
|
||||
_diff_db.close()
|
||||
except Exception as e:
|
||||
logger.warning("P84 diff-mode skipped: %s", e)
|
||||
|
||||
# B1 / B3 cross-cutting findings (own renderers, may be empty).
|
||||
reachability_html = state.get("reachability_html", "")
|
||||
retention_html = state.get("retention_html", "")
|
||||
|
||||
# Reihenfolge — Sales-optimiert.
|
||||
# B1 (Reachability) sits next to critical because it's an Art.7-Abs.3
|
||||
# finding. B3 (Retention) sits next to cookie_audit because both
|
||||
# are 3-source comparisons of cookie metadata.
|
||||
full_html = (
|
||||
gf_one_pager_html + audit_quality_html + input_warn_html
|
||||
+ bench_html + diff_html
|
||||
+ html_blocks["critical_html"] + reachability_html
|
||||
+ html_blocks["scope_disclaimer_html"]
|
||||
+ html_blocks["exec_summary_html"]
|
||||
+ html_blocks["cookie_arch_html"] + html_blocks["summary_html"]
|
||||
+ html_blocks["scanned_html"] + html_blocks["profile_html"]
|
||||
+ html_blocks["scorecard_html"] + internal_checks_html
|
||||
+ html_blocks["redundancy_html"]
|
||||
+ industry_ctx_html
|
||||
+ banner_shot_html
|
||||
+ html_blocks["providers_html"] + html_blocks["banner_deep_html"]
|
||||
+ html_blocks["cookie_audit_html"] + retention_html
|
||||
+ html_blocks["tcf_authority_html"]
|
||||
+ html_blocks["entropy_html"]
|
||||
+ html_blocks["network_trace_html"]
|
||||
+ html_blocks["library_mismatch_html"]
|
||||
+ html_blocks["consistency_html"] + html_blocks["signals_html"]
|
||||
+ html_blocks["solutions_html"]
|
||||
+ jc_decision_html
|
||||
+ html_blocks["vvt_html"] + html_blocks["report_html"]
|
||||
)
|
||||
|
||||
state["audit_quality_findings"] = audit_quality_findings
|
||||
state["full_html"] = full_html
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Phase D-3-Mid — Mid HTML blocks (P62/P103/P104/P105/audit/mismatch/signals).
|
||||
|
||||
Covers (in the original Step 5):
|
||||
- P62 Scope-Disclaimer
|
||||
- P103 Cookie-Value-Entropy + P104 Network-Tracing
|
||||
- P105 IAB TCF Authority cross-reference
|
||||
- Cookie-Compliance-Audit (3-Quellen-Vergleich, central USP)
|
||||
- P102 Cookie-Klassifikations-Pruefung (library mismatch)
|
||||
- P35/P77/P78 Doc-Text signals
|
||||
- P92/P94 Banner-Konsistenz
|
||||
- P73 MC-Solution-Generator (LLM suggestions per HIGH-Fail)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_d3_mid(state: dict) -> None:
|
||||
"""Mid HTML blocks. Mutates state in place."""
|
||||
doc_entries = state["doc_entries"]
|
||||
doc_texts = state["doc_texts"]
|
||||
banner_result = state["banner_result"]
|
||||
cmp_vendors = state["cmp_vendors"]
|
||||
fails_by_doc = state["fails_by_doc"]
|
||||
html_blocks = state["html_blocks"]
|
||||
|
||||
# P62: Marketing-Manager-Disclaimer
|
||||
scope_disclaimer_html = ""
|
||||
try:
|
||||
from ..scope_disclaimer import build_scope_disclaimer_html
|
||||
scope_disclaimer_html = build_scope_disclaimer_html()
|
||||
except Exception as e:
|
||||
logger.warning("Scope-disclaimer block skipped: %s", e)
|
||||
|
||||
# P103 + P104 — Cookie-Value-Entropy + Network-Tracing
|
||||
entropy_html = ""
|
||||
network_trace_html = ""
|
||||
try:
|
||||
from compliance.services.cookie_network_tracer import (
|
||||
build_network_trace_block_html,
|
||||
trace_cookie_network,
|
||||
)
|
||||
from compliance.services.cookie_value_entropy import (
|
||||
build_entropy_block_html,
|
||||
check_cookies_for_entropy_mismatch,
|
||||
)
|
||||
cookies_detailed = (banner_result or {}).get("cookies_detailed") or []
|
||||
entropy_findings = check_cookies_for_entropy_mismatch(cookies_detailed)
|
||||
if entropy_findings:
|
||||
entropy_html = build_entropy_block_html(entropy_findings)
|
||||
logger.info("P103 Entropy: %d Findings", len(entropy_findings))
|
||||
primary_url = ""
|
||||
for e_ in doc_entries:
|
||||
if e_.get("url"):
|
||||
primary_url = e_["url"]; break
|
||||
net_findings = trace_cookie_network(cookies_detailed, primary_url)
|
||||
if net_findings:
|
||||
network_trace_html = build_network_trace_block_html(net_findings)
|
||||
logger.info("P104 Network-Trace: %d Findings", len(net_findings))
|
||||
except Exception as e:
|
||||
logger.warning("P103/P104 entropy/network-trace skipped: %s", e)
|
||||
|
||||
# P105 — IAB TCF Authority-Cross-Reference
|
||||
tcf_authority_html = ""
|
||||
try:
|
||||
from compliance.services.tcf_vendor_authority import (
|
||||
build_tcf_authority_block_html, cross_reference_with_tcf,
|
||||
)
|
||||
from database import SessionLocal as _SLtcf
|
||||
_tcf_db = _SLtcf()
|
||||
try:
|
||||
tcf_findings = cross_reference_with_tcf(_tcf_db, cmp_vendors)
|
||||
if tcf_findings:
|
||||
tcf_authority_html = build_tcf_authority_block_html(tcf_findings)
|
||||
logger.info(
|
||||
"TCF-Authority: %d Vendor-Discrepancies gefunden",
|
||||
len(tcf_findings),
|
||||
)
|
||||
finally:
|
||||
_tcf_db.close()
|
||||
except Exception as e:
|
||||
logger.warning("TCF-Authority-Check skipped: %s", e)
|
||||
|
||||
# COOKIE-COMPLIANCE-AUDIT (3-Quellen-Vergleich — central USP)
|
||||
cookie_audit: dict = {}
|
||||
cookie_audit_html = ""
|
||||
try:
|
||||
from compliance.services.cookie_compliance_audit import (
|
||||
audit_cookie_compliance, build_cookie_audit_block_html,
|
||||
)
|
||||
from database import SessionLocal as _SLca
|
||||
_ca_db = _SLca()
|
||||
try:
|
||||
cookie_audit = audit_cookie_compliance(
|
||||
_ca_db, doc_texts.get("cookie") or doc_texts.get("dse"),
|
||||
banner_result,
|
||||
)
|
||||
if cookie_audit and (cookie_audit.get("declared_count") or
|
||||
cookie_audit.get("browser_count")):
|
||||
cookie_audit_html = build_cookie_audit_block_html(cookie_audit)
|
||||
logger.info(
|
||||
"Cookie-Audit: %d deklariert, %d im Browser, "
|
||||
"%d undokumentiert, %d compliant",
|
||||
cookie_audit.get("declared_count"),
|
||||
cookie_audit.get("browser_count"),
|
||||
len(cookie_audit.get("undeclared_in_browser") or []),
|
||||
len(cookie_audit.get("compliant") or []),
|
||||
)
|
||||
finally:
|
||||
_ca_db.close()
|
||||
except Exception as e:
|
||||
logger.warning("cookie-compliance-audit skipped: %s", e)
|
||||
|
||||
# P102: Cookie-Klassifikations-Pruefung
|
||||
library_mismatch_html = ""
|
||||
mismatches: list[dict] = []
|
||||
try:
|
||||
from compliance.services.cookie_library_mismatch import (
|
||||
build_mismatch_block_html, detect_mismatches,
|
||||
)
|
||||
from database import SessionLocal
|
||||
cookie_doc_for_check = doc_texts.get("cookie") or doc_texts.get("dse") or ""
|
||||
all_cookies_seen: list[str] = []
|
||||
if banner_result:
|
||||
for ph in (banner_result.get("phases") or {}).values():
|
||||
if isinstance(ph, dict):
|
||||
for ck in (ph.get("cookies") or []):
|
||||
if isinstance(ck, str):
|
||||
all_cookies_seen.append(ck)
|
||||
elif isinstance(ck, dict) and ck.get("name"):
|
||||
all_cookies_seen.append(ck["name"])
|
||||
if all_cookies_seen and cookie_doc_for_check:
|
||||
_mm_db = SessionLocal()
|
||||
try:
|
||||
mismatches = detect_mismatches(
|
||||
_mm_db, all_cookies_seen, cookie_doc_for_check,
|
||||
)
|
||||
if mismatches:
|
||||
library_mismatch_html = build_mismatch_block_html(mismatches)
|
||||
logger.info(
|
||||
"P102: %d Cookie-Mismatches gefunden", len(mismatches),
|
||||
)
|
||||
finally:
|
||||
_mm_db.close()
|
||||
except Exception as e:
|
||||
logger.warning("P102 mismatch detection failed: %s", e)
|
||||
|
||||
# P35 + P77 + P78: Textsignal-Checks
|
||||
signals_html = ""
|
||||
try:
|
||||
from compliance.services.doc_text_signals import (
|
||||
build_signals_block_html, run_all as run_signal_checks,
|
||||
)
|
||||
cookie_doc_missing = not bool(doc_texts.get("cookie"))
|
||||
sig_findings = run_signal_checks(
|
||||
banner_result, doc_texts, cookie_doc_missing,
|
||||
)
|
||||
if sig_findings:
|
||||
signals_html = build_signals_block_html(sig_findings)
|
||||
except Exception as e:
|
||||
logger.warning("P35/P77/P78 signals-check failed: %s", e)
|
||||
|
||||
# P92 + P94: Banner-Konsistenz
|
||||
consistency_html = ""
|
||||
try:
|
||||
from compliance.services.banner_consistency_checks import (
|
||||
build_consistency_block_html, run_all as run_consistency_checks,
|
||||
)
|
||||
cookie_doc_for_check = (doc_texts.get("cookie")
|
||||
or doc_texts.get("dse") or "")
|
||||
cons_findings = run_consistency_checks(
|
||||
banner_result or {}, cookie_doc_for_check, cmp_vendors,
|
||||
doc_texts=doc_texts,
|
||||
)
|
||||
if cons_findings:
|
||||
consistency_html = build_consistency_block_html(cons_findings)
|
||||
logger.info("P92/P94: %d Konsistenz-Findings", len(cons_findings))
|
||||
except Exception as e:
|
||||
logger.warning("P92/P94 consistency-check failed: %s", e)
|
||||
|
||||
# P73: MC-Solution-Generator — LLM-Vorschlaege pro HIGH-Fail
|
||||
solutions_html = ""
|
||||
try:
|
||||
from compliance.services.mc_solution_generator import (
|
||||
build_solutions_block_html, generate_solutions_for_fails,
|
||||
)
|
||||
all_solutions: list[dict] = []
|
||||
for dt, fails in fails_by_doc.items():
|
||||
if not fails:
|
||||
continue
|
||||
doc_txt = doc_texts.get(dt) or doc_texts.get("dse") or ""
|
||||
if not doc_txt or len(doc_txt) < 500:
|
||||
continue
|
||||
sols = await generate_solutions_for_fails(
|
||||
fails, doc_txt, dt, limit=3,
|
||||
)
|
||||
all_solutions.extend(sols)
|
||||
if len(all_solutions) >= 8:
|
||||
break
|
||||
if all_solutions:
|
||||
solutions_html = build_solutions_block_html(all_solutions[:8])
|
||||
logger.info("P73: %d MC-Solutions generiert", len(all_solutions))
|
||||
except Exception as e:
|
||||
logger.warning("P73 MC-Solution-Generator skipped: %s", e)
|
||||
|
||||
html_blocks.update({
|
||||
"scope_disclaimer_html": scope_disclaimer_html,
|
||||
"entropy_html": entropy_html,
|
||||
"network_trace_html": network_trace_html,
|
||||
"tcf_authority_html": tcf_authority_html,
|
||||
"cookie_audit_html": cookie_audit_html,
|
||||
"library_mismatch_html": library_mismatch_html,
|
||||
"signals_html": signals_html,
|
||||
"consistency_html": consistency_html,
|
||||
"solutions_html": solutions_html,
|
||||
})
|
||||
state["cookie_audit"] = cookie_audit
|
||||
state["mismatches"] = mismatches
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Phase D-3-Top — Top-of-mail HTML blocks.
|
||||
|
||||
Covers (in the original Step 5 of `_run_compliance_check`):
|
||||
- Summary / Scanned-URLs / Provider-list / Banner-deep / VVT HTML
|
||||
- MC-scorecard aggregation (all_mc_checks + scorecard) + trend lookup
|
||||
- P106 mc_audit_type split (internal_checks vs. verifiable_fails)
|
||||
- Profile HTML / Redundancy HTML
|
||||
- P1 Executive Summary
|
||||
- P18 Critical Findings block
|
||||
- P10 Cookie-Policy-Architecture detection
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ._helpers import _build_profile_html, _company_name_from_url, _extract_domain
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def run_phase_d3_top(state: dict) -> None:
|
||||
"""Top-of-mail HTML blocks. Mutates state in place."""
|
||||
req = state["req"]
|
||||
results = state["results"]
|
||||
doc_entries = state["doc_entries"]
|
||||
doc_texts = state["doc_texts"]
|
||||
banner_result = state["banner_result"]
|
||||
vvt_entries = state["vvt_entries"]
|
||||
cmp_vendors = state["cmp_vendors"]
|
||||
profile = state["profile"]
|
||||
redundancy_report = state.get("redundancy_report")
|
||||
|
||||
from ..agent_doc_check_banner import build_banner_deep_html
|
||||
from ..agent_doc_check_critical import build_critical_findings_html
|
||||
from ..agent_doc_check_exec_summary import build_exec_summary_html
|
||||
from ..agent_doc_check_extras import build_vvt_table_html
|
||||
from ..agent_doc_check_redundancy import build_redundancy_html
|
||||
from ..agent_doc_check_report import (
|
||||
build_html_report,
|
||||
build_management_summary,
|
||||
build_provider_list_html,
|
||||
build_scanned_urls_html,
|
||||
)
|
||||
from ..agent_doc_check_scorecard import build_scorecard_html
|
||||
from compliance.services.mc_scorecard import build_scorecard
|
||||
|
||||
summary_html = build_management_summary(results)
|
||||
scanned_html = build_scanned_urls_html(doc_entries)
|
||||
providers_html = build_provider_list_html(banner_result, vvt_entries)
|
||||
# P18: Deep-Block mit Phases + Quality-Score + Per-Category-Tracker
|
||||
banner_deep_html = build_banner_deep_html(banner_result)
|
||||
vvt_html = build_vvt_table_html(cmp_vendors)
|
||||
|
||||
# MC scorecard aggregated across ALL docs (DSGVO/TDDDG/BGB/...)
|
||||
all_mc_checks: list[dict] = []
|
||||
fails_by_doc: dict[str, list[dict]] = {}
|
||||
for r in results:
|
||||
for c in r.checks:
|
||||
if c.id.startswith("mc-"):
|
||||
rec = {
|
||||
"id": c.id, "label": c.label, "passed": c.passed,
|
||||
"severity": c.severity, "skipped": c.skipped,
|
||||
"regulation": c.regulation,
|
||||
"hint": getattr(c, "hint", "") or "",
|
||||
}
|
||||
all_mc_checks.append(rec)
|
||||
if (not c.passed and not c.skipped
|
||||
and (c.severity or "").upper() in ("CRITICAL", "HIGH")):
|
||||
fails_by_doc.setdefault(r.doc_type, []).append(rec)
|
||||
# P106 — Audit-Type-Klassifizierung pro MC
|
||||
mc_split: dict = {"internal_checks": [], "verifiable_fails": all_mc_checks}
|
||||
try:
|
||||
from compliance.services.mc_audit_type import (
|
||||
annotate_mc_results, split_by_audit_type,
|
||||
)
|
||||
annotate_mc_results(all_mc_checks)
|
||||
mc_split = split_by_audit_type(all_mc_checks)
|
||||
fails_by_doc = {}
|
||||
for r in mc_split.get("verifiable_fails") or []:
|
||||
fails_by_doc.setdefault("dse", []).append(r)
|
||||
except Exception as e:
|
||||
logger.warning("P106 mc_audit_type skipped: %s", e)
|
||||
scorecard = build_scorecard(all_mc_checks) if all_mc_checks else {}
|
||||
|
||||
# Trend: load previous scorecard for the same tenant + domain
|
||||
prev_scorecard: dict | None = None
|
||||
if scorecard:
|
||||
try:
|
||||
from compliance.services.compliance_audit_log import (
|
||||
list_runs_for_tenant,
|
||||
)
|
||||
tenant_id_for_trend = req.recipient or ""
|
||||
base_domain_for_trend = _extract_domain(doc_entries) or ""
|
||||
prev_runs = list_runs_for_tenant(
|
||||
tenant_id_for_trend,
|
||||
base_domain=base_domain_for_trend,
|
||||
limit=1,
|
||||
)
|
||||
if prev_runs:
|
||||
prev_scorecard = prev_runs[0].get("scorecard")
|
||||
except Exception as e:
|
||||
logger.debug("trend lookup skipped: %s", e)
|
||||
scorecard_html = (
|
||||
build_scorecard_html(scorecard, previous_scorecard=prev_scorecard)
|
||||
if scorecard else ""
|
||||
)
|
||||
|
||||
report_html = build_html_report(results, None, doc_texts)
|
||||
profile_html = _build_profile_html(profile)
|
||||
|
||||
# O4: Vendor-Redundanz / EU-Alternativen + Cost-Savings-Block
|
||||
redundancy_html = build_redundancy_html(redundancy_report)
|
||||
|
||||
# P1: Executive-Summary
|
||||
url_company_for_exec = _company_name_from_url(doc_entries)
|
||||
domain_for_exec = _extract_domain(doc_entries)
|
||||
site_name_for_exec = url_company_for_exec or domain_for_exec or ""
|
||||
exec_summary_html = build_exec_summary_html(
|
||||
scorecard=scorecard,
|
||||
previous_scorecard=prev_scorecard,
|
||||
cmp_vendors=cmp_vendors,
|
||||
redundancy_report=redundancy_report,
|
||||
site_name=site_name_for_exec,
|
||||
)
|
||||
|
||||
# P18: Critical-Findings-Block
|
||||
critical_html = ""
|
||||
try:
|
||||
critical_html = build_critical_findings_html(
|
||||
banner_result=banner_result,
|
||||
scorecard=scorecard,
|
||||
results=results,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("Critical-findings block skipped: %s", e)
|
||||
|
||||
# P10: Cookie-Policy-Architecture-Detection (BMW-Pattern erkennen)
|
||||
cookie_arch_html = ""
|
||||
try:
|
||||
from compliance.services.cookie_policy_architecture import (
|
||||
build_architecture_html,
|
||||
detect_architecture,
|
||||
)
|
||||
cookie_doc_url = ""
|
||||
cookie_doc_text = doc_texts.get("cookie", "")
|
||||
cookie_cmp_payloads: list[dict] = []
|
||||
for e in doc_entries:
|
||||
if (e.get("doc_type") or "").lower() in ("cookie", "cookie_policy"):
|
||||
cookie_doc_url = e.get("url", "")
|
||||
cookie_cmp_payloads = e.get("cmp_payloads") or []
|
||||
break
|
||||
# P17-A: Fallback wenn Cookie-Doc via P15 deduped wurde
|
||||
if not cookie_doc_text:
|
||||
dse_text = doc_texts.get("dse", "")
|
||||
if dse_text and any(w in dse_text.lower() for w in
|
||||
("cookie", "tracking", "google analytics",
|
||||
"consent")):
|
||||
cookie_doc_text = dse_text
|
||||
dse_entry = next((e for e in doc_entries
|
||||
if e.get("doc_type") == "dse"), {})
|
||||
cookie_doc_url = dse_entry.get("url", "")
|
||||
cookie_cmp_payloads = dse_entry.get("cmp_payloads") or []
|
||||
logger.info("P17-A: cookie-arch fallback auf DSE")
|
||||
if cookie_doc_text:
|
||||
arch = detect_architecture(
|
||||
doc_url=cookie_doc_url,
|
||||
doc_text=cookie_doc_text,
|
||||
cmp_payloads=cookie_cmp_payloads,
|
||||
homepage_cmp_payloads=state.get("cookie_payloads") or [],
|
||||
)
|
||||
cookie_arch_html = build_architecture_html(arch)
|
||||
logger.info("cookie-arch: layer=%s versioned=%s risk=%s",
|
||||
arch["layer_separation"], arch["versioned"],
|
||||
arch["risk_label"])
|
||||
except Exception as e:
|
||||
logger.warning("cookie-architecture detection failed: %s", e)
|
||||
|
||||
state["scorecard"] = scorecard
|
||||
state["prev_scorecard"] = prev_scorecard
|
||||
state["mc_split"] = mc_split
|
||||
state["fails_by_doc"] = fails_by_doc
|
||||
state["site_name_for_exec"] = site_name_for_exec
|
||||
state["domain_for_exec"] = domain_for_exec
|
||||
state["html_blocks"] = {
|
||||
"summary_html": summary_html,
|
||||
"scanned_html": scanned_html,
|
||||
"providers_html": providers_html,
|
||||
"banner_deep_html": banner_deep_html,
|
||||
"vvt_html": vvt_html,
|
||||
"scorecard_html": scorecard_html,
|
||||
"report_html": report_html,
|
||||
"profile_html": profile_html,
|
||||
"redundancy_html": redundancy_html,
|
||||
"exec_summary_html": exec_summary_html,
|
||||
"critical_html": critical_html,
|
||||
"cookie_arch_html": cookie_arch_html,
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Phase E — Send compliance-check email, with A1 ZIP-Anhang.
|
||||
|
||||
Original Step 6 of `_run_compliance_check`, extended with the A1
|
||||
attachment: when the Tesseract pipeline captured evidence slices,
|
||||
bundle them into evidence-{check_id}.zip (manifest.json +
|
||||
audit_metadata.json + slice_NNN.png) and attach to the e-mail. The
|
||||
attachment makes the evidence chain portable so a DSB / lawyer can
|
||||
hand it to an external auditor or supervisory authority.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from compliance.services.smtp_sender import send_email
|
||||
|
||||
from ._helpers import _company_name_from_url, _extract_domain, _update
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_phase_e(state: dict) -> None:
|
||||
"""Build site label, optional ZIP attachment, send mail. Mutate state."""
|
||||
check_id = state["check_id"]
|
||||
req = state["req"]
|
||||
results = state["results"]
|
||||
doc_entries = state["doc_entries"]
|
||||
full_html = state["full_html"]
|
||||
cookie_evidence_slices = state.get("cookie_evidence_slices")
|
||||
cookie_evidence_meta = state.get("cookie_evidence_meta")
|
||||
|
||||
# Derive site name primarily from entered URL.
|
||||
# The extracted_profile.companyName is often noisy (e.g. picks up
|
||||
# juris.de from legal references). Domain-derived name is more
|
||||
# predictable for the GF email subject.
|
||||
doc_count = len([r for r in results if not r.error])
|
||||
url_company = _company_name_from_url(doc_entries)
|
||||
domain = _extract_domain(doc_entries)
|
||||
site_name = url_company or domain or "Unbekannt"
|
||||
_update(check_id, "E-Mail wird versendet...", 98)
|
||||
|
||||
# A1: bundle cookie-evidence slices into a ZIP attachment so the
|
||||
# audit chain reaches the recipient. Each slice has its own
|
||||
# SHA-256 + capture timestamp; manifest.json + audit_metadata.json
|
||||
# make the chain verifiable for an external auditor.
|
||||
evidence_attachments: list[dict] = []
|
||||
if cookie_evidence_slices:
|
||||
try:
|
||||
from compliance.services.evidence_zip_builder import (
|
||||
build_evidence_zip,
|
||||
)
|
||||
zip_bytes = build_evidence_zip(
|
||||
slices=cookie_evidence_slices,
|
||||
meta=cookie_evidence_meta,
|
||||
check_id=check_id,
|
||||
)
|
||||
evidence_attachments.append({
|
||||
"filename": f"evidence-{check_id[:8]}.zip",
|
||||
"data": zip_bytes,
|
||||
"mime": "application/zip",
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("A1 evidence-zip build failed: %s", e)
|
||||
|
||||
email_result = send_email(
|
||||
recipient=req.recipient,
|
||||
subject=f"[COMPLIANCE-CHECK] {site_name} — {doc_count} Dokumente geprueft",
|
||||
body_html=full_html,
|
||||
attachments=evidence_attachments or None,
|
||||
)
|
||||
|
||||
state["email_result"] = email_result
|
||||
state["site_name"] = site_name
|
||||
state["domain"] = domain
|
||||
state["doc_count"] = doc_count
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Phase F — Build response + persist snapshot/audit-log/unified-findings.
|
||||
|
||||
Covers (in the original `_run_compliance_check`):
|
||||
- Step 7 Build response dict, mark job as completed
|
||||
- P80 Persist raw scan data so we can replay the audit pipeline
|
||||
without re-crawling (7min → 5sec test cycle)
|
||||
- SQLite audit log (compliance.api/audit endpoints + trend view A6)
|
||||
- P5 Unified findings (MC + Pflichtangaben + Vendor + Redundanz
|
||||
in one searchable table behind /agent/findings/<id>)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ._constants import _compliance_check_jobs
|
||||
from ._helpers import _result_to_dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_phase_f(state: dict) -> None:
|
||||
"""Build response + persist. Mutates state in place."""
|
||||
check_id = state["check_id"]
|
||||
req = state["req"]
|
||||
results = state["results"]
|
||||
profile = state["profile"]
|
||||
profile_dict = state["profile_dict"]
|
||||
extracted_profile = state["extracted_profile"]
|
||||
banner_result = state["banner_result"]
|
||||
tcf_vendors = state["tcf_vendors"]
|
||||
vvt_entries = state["vvt_entries"]
|
||||
cmp_vendors = state["cmp_vendors"]
|
||||
cookie_audit = state["cookie_audit"]
|
||||
total_findings = state["total_findings"]
|
||||
email_result = state["email_result"]
|
||||
doc_entries = state["doc_entries"]
|
||||
doc_texts = state["doc_texts"]
|
||||
redundancy_report = state.get("redundancy_report")
|
||||
scorecard = state["scorecard"]
|
||||
site_name = state.get("site_name", "")
|
||||
domain = state.get("domain", "")
|
||||
doc_count = state.get("doc_count", 0)
|
||||
|
||||
response = {
|
||||
"check_id": check_id,
|
||||
"results": [_result_to_dict(r) for r in results],
|
||||
"business_profile": profile_dict,
|
||||
"extracted_profile": extracted_profile,
|
||||
# P18: vollen consent-tester-Output durchreichen statt nur 4 Felder.
|
||||
# phases (before/after-accept/reject) + banner_checks.violations +
|
||||
# category_tests werden vom Renderer + Critical-Findings-Block genutzt.
|
||||
"banner_result": ({
|
||||
"detected": banner_result.get("banner_detected", False),
|
||||
"provider": banner_result.get("banner_provider", ""),
|
||||
"violations": len((banner_result.get("banner_checks") or {})
|
||||
.get("violations", [])),
|
||||
"tcf_vendor_count": len(tcf_vendors),
|
||||
"completeness_pct": banner_result.get("completeness_pct"),
|
||||
"correctness_pct": banner_result.get("correctness_pct"),
|
||||
"phases": banner_result.get("phases", {}),
|
||||
"banner_checks": banner_result.get("banner_checks", {}),
|
||||
"category_tests": banner_result.get("category_tests", []),
|
||||
"structured_checks": banner_result.get("structured_checks", []),
|
||||
"summary": banner_result.get("summary", {}),
|
||||
} if banner_result else None),
|
||||
"tcf_vendors": vvt_entries if tcf_vendors else [],
|
||||
"cmp_vendors": cmp_vendors,
|
||||
"cookie_audit": cookie_audit if cookie_audit else None,
|
||||
"total_documents": len(results),
|
||||
"total_findings": total_findings,
|
||||
"email_status": email_result.get("status", "failed"),
|
||||
"checked_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
_compliance_check_jobs[check_id]["status"] = "completed"
|
||||
_compliance_check_jobs[check_id]["result"] = response
|
||||
_compliance_check_jobs[check_id]["progress"] = "Fertig"
|
||||
_compliance_check_jobs[check_id]["progress_pct"] = 100
|
||||
|
||||
# P80: persist raw scan data so we can replay audit pipeline
|
||||
# without re-crawling (7min -> 5sec test cycle).
|
||||
try:
|
||||
from database import SessionLocal
|
||||
from compliance.services.check_snapshot import save_snapshot
|
||||
snap_db = SessionLocal()
|
||||
try:
|
||||
save_snapshot(
|
||||
snap_db,
|
||||
check_id=check_id,
|
||||
doc_entries=doc_entries,
|
||||
banner_result=banner_result,
|
||||
profile=profile,
|
||||
cmp_vendors=cmp_vendors,
|
||||
scan_context=req.scan_context, # P79
|
||||
site_label=site_name,
|
||||
notes=f"recipient={req.recipient}",
|
||||
)
|
||||
finally:
|
||||
snap_db.close()
|
||||
except Exception as snap_err:
|
||||
logger.warning("P80 snapshot save skipped: %s", snap_err)
|
||||
|
||||
# Persist to sidecar SQLite audit log — enables /audit endpoints
|
||||
# (A5 admin tab) and trend view (A6). Best-effort; failures here
|
||||
# do not affect the user-facing response.
|
||||
try:
|
||||
from compliance.services.compliance_audit_log import record_check_run
|
||||
from compliance.services.mc_scorecard import full_audit_records
|
||||
audit_rows: list[dict] = []
|
||||
for r in results:
|
||||
doc_mc = [c for c in r.checks if c.id.startswith("mc-")]
|
||||
audit_rows.extend(full_audit_records(
|
||||
[{"id": c.id, "label": c.label, "passed": c.passed,
|
||||
"severity": c.severity, "skipped": c.skipped,
|
||||
"regulation": c.regulation, "matched_text": c.matched_text,
|
||||
"hint": c.hint, "level": c.level}
|
||||
for c in doc_mc],
|
||||
check_id=check_id,
|
||||
doc_type=r.doc_type,
|
||||
))
|
||||
record_check_run(
|
||||
check_id=check_id,
|
||||
tenant_id=req.recipient or "",
|
||||
site_name=site_name,
|
||||
base_domain=domain or "",
|
||||
doc_count=doc_count,
|
||||
scorecard=scorecard,
|
||||
vvt_summary={
|
||||
"total": len(cmp_vendors),
|
||||
"internal": sum(1 for v in cmp_vendors
|
||||
if (v.get("recipient_type") or "").upper()
|
||||
in ("INTERNAL", "GROUP_COMPANY")),
|
||||
"external": sum(1 for v in cmp_vendors
|
||||
if (v.get("recipient_type") or "").upper()
|
||||
in ("PROCESSOR", "CONTROLLER")),
|
||||
},
|
||||
mc_records=audit_rows,
|
||||
)
|
||||
from compliance.services.compliance_audit_log import record_check_payload
|
||||
record_check_payload(
|
||||
check_id=check_id,
|
||||
vendors=cmp_vendors,
|
||||
profile=extracted_profile,
|
||||
banner=banner_result,
|
||||
)
|
||||
# Unified findings (P5): bundle MC + Pflichtangaben + Vendor +
|
||||
# Redundanz in one searchable table behind /agent/findings/<id>.
|
||||
try:
|
||||
from compliance.services.unified_findings_collector import collect
|
||||
from compliance.services.unified_findings_store import record_findings
|
||||
unified = collect(
|
||||
check_id=check_id,
|
||||
results=results,
|
||||
cmp_vendors=cmp_vendors,
|
||||
redundancy_report=redundancy_report,
|
||||
doc_texts=doc_texts,
|
||||
)
|
||||
record_findings(check_id, unified)
|
||||
except Exception as e:
|
||||
logger.warning("Unified findings collect failed: %s", e)
|
||||
except Exception as e:
|
||||
logger.warning("Audit persistence skipped: %s", e)
|
||||
|
||||
state["response"] = response
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Pydantic request/response schemas for the compliance-check route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ExtractTextRequest(BaseModel):
|
||||
url: str
|
||||
|
||||
|
||||
class DocumentInput(BaseModel):
|
||||
doc_type: str # dse, agb, impressum, cookie, widerruf, avv, loeschkonzept, etc.
|
||||
url: str = ""
|
||||
text: str = "" # text has priority over URL
|
||||
|
||||
|
||||
class ComplianceCheckRequest(BaseModel):
|
||||
documents: list[DocumentInput]
|
||||
use_agent: bool = False
|
||||
recipient: str = "dsb@breakpilot.local"
|
||||
# P12: Override fuer TDM-Vorbehalt bei dokumentierter Kunden-Erlaubnis.
|
||||
# Pflichtfeld tdm_override_reason wenn tdm_override=True
|
||||
# (z.B. "Auftragsbeziehung Safetykon GmbH, Email Hr. X 18.05.2026").
|
||||
tdm_override: bool = False
|
||||
tdm_override_reason: str = ""
|
||||
# P79: 8-Feld Pre-Scan-Wizard (Branche, B2B/B2C, Direkt-Vertrieb,
|
||||
# Rechtsform, Konzern, MA, Besondere Daten, Drittland). Wird im
|
||||
# Snapshot persistiert und filtert die MC-Auswertung (P72).
|
||||
scan_context: dict | None = None
|
||||
|
||||
|
||||
class ComplianceCheckStartResponse(BaseModel):
|
||||
check_id: str
|
||||
status: str = "running"
|
||||
|
||||
|
||||
class ComplianceCheckStatusResponse(BaseModel):
|
||||
check_id: str
|
||||
status: str
|
||||
progress: str = ""
|
||||
progress_pct: int = 0
|
||||
result: dict | None = None
|
||||
error: str = ""
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Per-document regex + MC + LLM checks for the compliance-check route.
|
||||
|
||||
Each document goes through:
|
||||
1. regex completeness/correctness checklist
|
||||
2. Master Control evaluation (all MCs for this doc_type)
|
||||
3. LLM verification of failed regex checks (overturns where evidence
|
||||
was missed by the regex)
|
||||
4. Cookie-only: opt-out + privacy-policy URL health-check
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def _check_single(
|
||||
text: str, doc_type: str, label: str, url: str,
|
||||
word_count: int, use_agent: bool,
|
||||
business_scope: set[str] | None = None,
|
||||
business_profile: dict | None = None,
|
||||
):
|
||||
"""Run regex + MC checks on a single document."""
|
||||
from compliance.services.doc_checks.runner import check_document_completeness
|
||||
from compliance.services.rag_document_checker import check_document_with_controls
|
||||
from ..agent_doc_check_routes import CheckItem, DocCheckResult
|
||||
|
||||
# Regex checklist
|
||||
findings = check_document_completeness(text, doc_type, label, url,
|
||||
business_profile=business_profile)
|
||||
|
||||
all_checks: list[CheckItem] = []
|
||||
completeness = 0
|
||||
correctness = 0
|
||||
|
||||
for f in findings:
|
||||
if "SCORE" in f.get("code", ""):
|
||||
for c in f.get("all_checks", []):
|
||||
all_checks.append(CheckItem(
|
||||
id=c["id"], label=c["label"], passed=c["passed"],
|
||||
severity=c["severity"], matched_text=c.get("matched_text", ""),
|
||||
level=c.get("level", 1), parent=c.get("parent"),
|
||||
skipped=c.get("skipped", False), hint=c.get("hint", ""),
|
||||
))
|
||||
completeness = f.get("completeness_pct", 0)
|
||||
correctness = f.get("correctness_pct", 0)
|
||||
|
||||
# Master Control checks (top 20 by severity to avoid noise)
|
||||
try:
|
||||
# max_controls=0 -> evaluate ALL MCs for this doc_type (DB has
|
||||
# 1874 across 8 types; regex matching is cheap and dominates
|
||||
# well under 1s per doc). Caps remain on the LLM-enrich step
|
||||
# (top-10 FAILs) so cost stays bounded.
|
||||
mc_results = await check_document_with_controls(
|
||||
text, doc_type, label, max_controls=0, use_agent=use_agent,
|
||||
business_scope=business_scope,
|
||||
)
|
||||
if mc_results:
|
||||
for mc in mc_results:
|
||||
all_checks.append(CheckItem(**mc))
|
||||
l2 = [c for c in all_checks if c.level == 2 and not c.skipped]
|
||||
l2_passed = sum(1 for c in l2 if c.passed)
|
||||
correctness = round(l2_passed / len(l2) * 100) if l2 else 0
|
||||
except Exception as e:
|
||||
logger.warning("MC check skipped for %s: %s", label, e)
|
||||
|
||||
# LLM verification of regex fails
|
||||
failed = [c for c in all_checks if not c.passed and not c.skipped and c.hint]
|
||||
if failed:
|
||||
try:
|
||||
from compliance.services.doc_checks.llm_verify import verify_failed_checks
|
||||
overturns = await verify_failed_checks(
|
||||
text,
|
||||
[{"id": c.id, "label": c.label, "hint": c.hint} for c in failed],
|
||||
label,
|
||||
)
|
||||
for c in all_checks:
|
||||
if c.id in overturns and overturns[c.id]["overturned"]:
|
||||
c.passed = True
|
||||
c.matched_text = f"[LLM] {overturns[c.id]['evidence']}"
|
||||
l2_active = [c for c in all_checks if c.level == 2 and not c.skipped]
|
||||
l2_passed = sum(1 for c in l2_active if c.passed)
|
||||
if l2_active:
|
||||
correctness = round(l2_passed / len(l2_active) * 100)
|
||||
except Exception as e:
|
||||
logger.warning("LLM verification skipped: %s", e)
|
||||
|
||||
# Cookie-policy only: actively HTTP-probe the Opt-Out + Privacy-Policy
|
||||
# URLs the document advertises. Broken links make individual provider
|
||||
# entries non-compliant under Art. 7(3) DSGVO.
|
||||
if doc_type == "cookie":
|
||||
try:
|
||||
from compliance.services.cookie_link_validator import (
|
||||
extract_links, validate_links, build_check_items,
|
||||
)
|
||||
links = extract_links(text)
|
||||
if links:
|
||||
logger.info("Cookie-link validator: %d urls extracted from %s",
|
||||
len(links), label)
|
||||
validated = await validate_links(links)
|
||||
for item in build_check_items(validated):
|
||||
all_checks.append(CheckItem(**item))
|
||||
# Re-compute correctness with the new L2 items
|
||||
l2_active = [c for c in all_checks if c.level == 2 and not c.skipped]
|
||||
l2_passed = sum(1 for c in l2_active if c.passed)
|
||||
if l2_active:
|
||||
correctness = round(l2_passed / len(l2_active) * 100)
|
||||
except Exception as e:
|
||||
logger.warning("Cookie-link validation skipped for %s: %s", label, e)
|
||||
|
||||
non_score = [f for f in findings if "SCORE" not in f.get("code", "")]
|
||||
return DocCheckResult(
|
||||
label=label, url=url, doc_type=doc_type,
|
||||
word_count=word_count or len(text.split()),
|
||||
completeness_pct=completeness, correctness_pct=correctness,
|
||||
checks=all_checks, findings_count=len(non_score),
|
||||
)
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Shared state for the compliance-check pipeline.
|
||||
|
||||
The 7-step pipeline accumulates ~60 named values that flow across
|
||||
phases (doc_entries, profile, results, banner_result, cmp_vendors,
|
||||
scorecard, HTML blocks, …). Rather than threading 60 parameters
|
||||
through each function, we pass one mutable `CheckState` dict.
|
||||
|
||||
Phases read what they need with `state[key]` and write their outputs
|
||||
with `state[key] = value`. This is intentionally untyped: enforcing
|
||||
strict typing would require freezing the schema before all phases
|
||||
landed, and the report-building phase routinely adds new optional
|
||||
keys (P1, P10, P50, P59b, P82, P103, P104, P106, …).
|
||||
|
||||
`CheckState.new(check_id, req)` initialises the dict with the few
|
||||
keys that must exist from the start.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def new_state(check_id: str, req) -> dict:
|
||||
"""Create a fresh state dict for a check run.
|
||||
|
||||
Pre-populates a few keys that downstream phases assume exist
|
||||
(e.g. `cmp_vendors` defaulting to `[]`).
|
||||
"""
|
||||
return {
|
||||
"check_id": check_id,
|
||||
"req": req,
|
||||
# Phase-1 outputs
|
||||
"doc_texts": {},
|
||||
"doc_entries": [],
|
||||
"url_text_cache": {},
|
||||
"pasted_table_vendors": [],
|
||||
"placement_findings": [],
|
||||
# Phase-2/3/4 outputs
|
||||
"profile": None,
|
||||
"profile_dict": {},
|
||||
"results": [],
|
||||
"total_findings": 0,
|
||||
"business_scope": set(),
|
||||
"banner_result": None,
|
||||
"banner_url": "",
|
||||
"tcf_vendors": [],
|
||||
"vvt_entries": [],
|
||||
"extracted_profile": {},
|
||||
# Phase-5 outputs
|
||||
"cmp_vendors": [],
|
||||
"cookie_audit": {},
|
||||
"cookie_evidence_slices": None,
|
||||
"cookie_evidence_meta": None,
|
||||
"scorecard": {},
|
||||
"full_html": "",
|
||||
"audit_quality_findings": [],
|
||||
# Phase-6/7 outputs
|
||||
"email_result": {"status": "skipped"},
|
||||
"site_name": "",
|
||||
}
|
||||
Reference in New Issue
Block a user