feat(compliance-check): exec-summary + voll-audit + TDM-respect + cookie-KB-extended + saving-scan-funnel
CI / detect-changes (push) Successful in 10s
CI / branch-name (push) Has been skipped
CI / guardrail-integrity (push) Has been skipped
CI / secret-scan (push) Has been skipped
CI / dep-audit (push) Has been skipped
CI / sbom-scan (push) Has been skipped
CI / validate-canonical-controls (push) Successful in 14s
CI / loc-budget (push) Failing after 15s
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / nodejs-build (push) Successful in 2m43s
CI / test-go (push) Has been skipped
CI / iace-gt-coverage (push) Has been skipped
CI / test-python-backend (push) Successful in 37s
CI / test-python-document-crawler (push) Has been skipped
CI / test-python-dsms-gateway (push) Has been skipped
CI / detect-changes (push) Successful in 10s
CI / branch-name (push) Has been skipped
CI / guardrail-integrity (push) Has been skipped
CI / secret-scan (push) Has been skipped
CI / dep-audit (push) Has been skipped
CI / sbom-scan (push) Has been skipped
CI / validate-canonical-controls (push) Successful in 14s
CI / loc-budget (push) Failing after 15s
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / nodejs-build (push) Successful in 2m43s
CI / test-go (push) Has been skipped
CI / iace-gt-coverage (push) Has been skipped
CI / test-python-backend (push) Successful in 37s
CI / test-python-document-crawler (push) Has been skipped
CI / test-python-dsms-gateway (push) Has been skipped
P1 — Exec-Summary oben im Email-Report (4 KPIs + 2 CTAs, dunkler Gradient)
P3 — no_direct_sales-Flag fuer OEM-Konfigurator-Sites; AGB/Widerruf/AGB als
"NICHT ANWENDBAR" (grau) statt "NICHT GEFUNDEN" (rot)
P5 — Voll-Audit Unification: alle Findings (MC + Pflichtangaben + Vendor +
Redundanz) in /data/compliance_audits.db.unified_findings; neuer
/api/compliance/agent/findings/<id> Endpoint + FindingsTab im Audit-UI
mit Filter + CSV-Export
P7 — Crawl-Hardening: TDM-Reservation-Check (robots.txt / ai.txt / Header /
Meta) vor jedem Run mit 24h-Cache; HeadlessChrome-UA (Firma noch nicht
gegruendet — Switch via BREAKPILOT_BRANDED_UA env); per-Domain
Rate-Limit 1 req/s + max 2 concurrent
P2 — Cookie-Knowledge-DB additiv erweitert (35 -> 74 Cookies): Adobe, Meta,
Microsoft, LinkedIn, TikTok, HubSpot, Marketo, Salesforce, Hotjar,
FullStory, Mouseflow, Intercom, Drift, Zendesk, Cloudflare, Stripe,
OneTrust/Cookiebot/Usercentrics, Matomo, Pinterest, Snapchat, X/Twitter,
YouTube, Vimeo, Klaviyo, Mailchimp, Mixpanel, Segment, Amplitude,
Optimizely, Datadog; Wire-in in cookie_function_classifier liefert
compliance_risk-Label (kritisch/hoch/mittel/gering) pro Vendor
A — k-Anonymitaets-Helper (benchmark_k_anonymity) fuer P6-Vorbereitung
B — Cross-Tenant-Domain-Assertion im /findings-Endpoint (expected_domain
Query-Param -> 403 bei Mismatch)
C — Saving-Scan-Funnel: /api/compliance/agent/saving-scan/start mit
Validierung + 24h-Rate-Limit pro Domain + Lead-Persistenz in
saving_scan_leads + Auto-Discovery via _run_compliance_check; 6 Tests
D — Risk-Badge im Email-Vendor-Row
Rechtliche Leitplanken (Memory feedback_oem_data_legal.md): nur eigene
Knapp-Bewertungen + Source-Pointer, keine 1:1-Kopien fremder CMP-Texte.
TDM-Opt-Out-Respect nach § 44b UrhG. KEINE Schema-Aenderungen — alles in
Sidecar-SQLite.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,28 @@
|
|||||||
|
/**
|
||||||
|
* Proxy: GET /api/sdk/v1/agent/findings/<checkId>
|
||||||
|
* -> backend GET /api/compliance/agent/findings/<checkId>
|
||||||
|
*
|
||||||
|
* Forwards all query params (source, severity, doc_type, status, q, limit).
|
||||||
|
*/
|
||||||
|
import { NextRequest, NextResponse } from 'next/server'
|
||||||
|
|
||||||
|
const BACKEND_URL = process.env.BACKEND_API_URL || 'http://backend-compliance:8002'
|
||||||
|
|
||||||
|
export async function GET(
|
||||||
|
request: NextRequest,
|
||||||
|
{ params }: { params: { checkId: string } },
|
||||||
|
) {
|
||||||
|
const checkId = params.checkId
|
||||||
|
const qs = request.nextUrl.searchParams.toString()
|
||||||
|
const url = `${BACKEND_URL}/api/compliance/agent/findings/${checkId}${qs ? `?${qs}` : ''}`
|
||||||
|
try {
|
||||||
|
const resp = await fetch(url, { signal: AbortSignal.timeout(20000) })
|
||||||
|
const data = await resp.json()
|
||||||
|
return NextResponse.json(data, { status: resp.status })
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Findings-Abfrage fehlgeschlagen' },
|
||||||
|
{ status: 503 },
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -119,11 +119,9 @@ export function ComplianceCheckTab() {
|
|||||||
localStorage.removeItem(STORAGE_KEY_CHECK_ID); setActiveCheckId('')
|
localStorage.removeItem(STORAGE_KEY_CHECK_ID); setActiveCheckId('')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (data.status === 'failed' || data.status === 'not_found') {
|
if (['failed', 'not_found', 'skipped_tdm'].includes(data.status)) {
|
||||||
if (data.status === 'failed') setError(data.error || 'Pruefung fehlgeschlagen')
|
if (data.status !== 'not_found') setError(data.error || (data.status === 'skipped_tdm' ? 'TDM-Vorbehalt erkannt — Crawl uebersprungen' : 'Pruefung fehlgeschlagen'))
|
||||||
setProgress(''); setProgressPct(0); setLoading(false)
|
setProgress(''); setProgressPct(0); setLoading(false); localStorage.removeItem(STORAGE_KEY_CHECK_ID); setActiveCheckId(''); return
|
||||||
localStorage.removeItem(STORAGE_KEY_CHECK_ID); setActiveCheckId('')
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
} catch { /* retry */ }
|
} catch { /* retry */ }
|
||||||
}
|
}
|
||||||
@@ -236,9 +234,9 @@ export function ComplianceCheckTab() {
|
|||||||
localStorage.setItem(STORAGE_KEY_HISTORY, JSON.stringify(updated))
|
localStorage.setItem(STORAGE_KEY_HISTORY, JSON.stringify(updated))
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if (pollData.status === 'failed') {
|
if (['failed', 'skipped_tdm'].includes(pollData.status)) {
|
||||||
localStorage.removeItem(STORAGE_KEY_CHECK_ID); setActiveCheckId('')
|
localStorage.removeItem(STORAGE_KEY_CHECK_ID); setActiveCheckId('')
|
||||||
throw new Error(pollData.error || 'Pruefung fehlgeschlagen')
|
throw new Error(pollData.error || (pollData.status === 'skipped_tdm' ? 'TDM-Vorbehalt' : 'Pruefung fehlgeschlagen'))
|
||||||
}
|
}
|
||||||
attempts++
|
attempts++
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,274 @@
|
|||||||
|
'use client'
|
||||||
|
|
||||||
|
import React, { useEffect, useMemo, useState } from 'react'
|
||||||
|
|
||||||
|
type Finding = {
|
||||||
|
id: number
|
||||||
|
source_type: string
|
||||||
|
doc_type: string
|
||||||
|
severity: string
|
||||||
|
status: string
|
||||||
|
regulation: string
|
||||||
|
label: string
|
||||||
|
hint: string
|
||||||
|
action_recipe: Record<string, string>
|
||||||
|
anchor_excerpt: string
|
||||||
|
anchor_conf: number
|
||||||
|
vendor_name: string
|
||||||
|
category: string
|
||||||
|
payload: Record<string, unknown>
|
||||||
|
}
|
||||||
|
|
||||||
|
type Summary = {
|
||||||
|
total: number
|
||||||
|
by_source: Record<string, number>
|
||||||
|
by_severity: Record<string, number>
|
||||||
|
by_status: Record<string, number>
|
||||||
|
by_doc_type: Record<string, number>
|
||||||
|
}
|
||||||
|
|
||||||
|
type Resp = {
|
||||||
|
found: boolean
|
||||||
|
summary: Summary
|
||||||
|
count: number
|
||||||
|
findings: Finding[]
|
||||||
|
}
|
||||||
|
|
||||||
|
const SOURCE_LABEL: Record<string, string> = {
|
||||||
|
all: 'Alle Quellen',
|
||||||
|
mc: 'Master-Controls',
|
||||||
|
pflichtangabe: 'Pflichtangaben',
|
||||||
|
vendor: 'Vendor-Findings',
|
||||||
|
redundanz: 'Redundanzen',
|
||||||
|
}
|
||||||
|
|
||||||
|
const SEVERITY_COLOR: Record<string, string> = {
|
||||||
|
CRITICAL: 'bg-red-600 text-white',
|
||||||
|
HIGH: 'bg-red-100 text-red-800',
|
||||||
|
MEDIUM: 'bg-amber-100 text-amber-800',
|
||||||
|
LOW: 'bg-blue-100 text-blue-800',
|
||||||
|
INFO: 'bg-gray-100 text-gray-600',
|
||||||
|
}
|
||||||
|
|
||||||
|
const STATUS_LABEL: Record<string, string> = {
|
||||||
|
failed: 'Fail',
|
||||||
|
passed: 'Pass',
|
||||||
|
skipped: 'Skip',
|
||||||
|
na: 'N/A',
|
||||||
|
info: 'Info',
|
||||||
|
}
|
||||||
|
|
||||||
|
const SEVERITY_OPTS = ['all', 'CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'INFO']
|
||||||
|
const STATUS_OPTS = ['all', 'failed', 'passed', 'skipped', 'na', 'info']
|
||||||
|
|
||||||
|
export default function FindingsTab({ checkId }: { checkId: string }) {
|
||||||
|
const [data, setData] = useState<Resp | null>(null)
|
||||||
|
const [loading, setLoading] = useState(true)
|
||||||
|
const [error, setError] = useState<string | null>(null)
|
||||||
|
const [source, setSource] = useState('all')
|
||||||
|
const [severity, setSeverity] = useState('all')
|
||||||
|
const [docType, setDocType] = useState('all')
|
||||||
|
const [status, setStatus] = useState('failed')
|
||||||
|
const [q, setQ] = useState('')
|
||||||
|
const [expanded, setExpanded] = useState<number | null>(null)
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false
|
||||||
|
setLoading(true)
|
||||||
|
const qs = new URLSearchParams({
|
||||||
|
source, severity, doc_type: docType, status, q, limit: '1500',
|
||||||
|
}).toString()
|
||||||
|
fetch(`/api/sdk/v1/agent/findings/${checkId}?${qs}`)
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => { if (!cancelled) setData(d) })
|
||||||
|
.catch(e => { if (!cancelled) setError(String(e)) })
|
||||||
|
.finally(() => { if (!cancelled) setLoading(false) })
|
||||||
|
return () => { cancelled = true }
|
||||||
|
}, [checkId, source, severity, docType, status, q])
|
||||||
|
|
||||||
|
const docTypes = useMemo(
|
||||||
|
() => Object.keys(data?.summary?.by_doc_type ?? {}).filter(d => d !== '-').sort(),
|
||||||
|
[data],
|
||||||
|
)
|
||||||
|
|
||||||
|
const csvExport = () => {
|
||||||
|
const rows = data?.findings ?? []
|
||||||
|
const head = ['Quelle', 'Doc', 'Severity', 'Status', 'Regulation', 'Label', 'Vendor', 'Hint']
|
||||||
|
const lines = [head.join(',')]
|
||||||
|
for (const r of rows) {
|
||||||
|
const cells = [
|
||||||
|
r.source_type, r.doc_type, r.severity, r.status,
|
||||||
|
r.regulation, r.label, r.vendor_name, r.hint,
|
||||||
|
].map(c => `"${String(c ?? '').replace(/"/g, '""').replace(/\n/g, ' ')}"`)
|
||||||
|
lines.push(cells.join(','))
|
||||||
|
}
|
||||||
|
const blob = new Blob([lines.join('\n')], { type: 'text/csv;charset=utf-8' })
|
||||||
|
const url = URL.createObjectURL(blob)
|
||||||
|
const a = document.createElement('a')
|
||||||
|
a.href = url
|
||||||
|
a.download = `findings-${checkId}.csv`
|
||||||
|
a.click()
|
||||||
|
URL.revokeObjectURL(url)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (loading && !data) return <div className="p-6 text-sm text-gray-500">Lade Voll-Audit…</div>
|
||||||
|
if (error) return <div className="p-6 text-sm text-red-600">Fehler: {error}</div>
|
||||||
|
if (!data?.found) {
|
||||||
|
return (
|
||||||
|
<div className="p-6 text-sm text-gray-500">
|
||||||
|
Keine unified findings für diesen Run gespeichert (alter Run vor P5?).
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const sum = data.summary
|
||||||
|
const findings = data.findings
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="space-y-4">
|
||||||
|
{/* Summary Cards */}
|
||||||
|
<div className="grid grid-cols-2 md:grid-cols-4 gap-3 text-xs">
|
||||||
|
{Object.entries(SOURCE_LABEL).filter(([k]) => k !== 'all').map(([k, label]) => {
|
||||||
|
const count = sum.by_source?.[k] ?? 0
|
||||||
|
return (
|
||||||
|
<button key={k}
|
||||||
|
onClick={() => setSource(source === k ? 'all' : k)}
|
||||||
|
className={`text-left rounded-lg border px-3 py-2 transition ${
|
||||||
|
source === k
|
||||||
|
? 'border-blue-500 bg-blue-50 text-blue-900'
|
||||||
|
: 'border-gray-200 hover:border-gray-300 bg-white'
|
||||||
|
}`}>
|
||||||
|
<div className="text-[10px] uppercase tracking-wide text-gray-500">{label}</div>
|
||||||
|
<div className="text-lg font-semibold">{count}</div>
|
||||||
|
</button>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Filter row */}
|
||||||
|
<div className="flex flex-wrap gap-2 items-center text-xs">
|
||||||
|
<select value={severity} onChange={e => setSeverity(e.target.value)}
|
||||||
|
className="border border-gray-200 rounded px-2 py-1">
|
||||||
|
{SEVERITY_OPTS.map(s => (
|
||||||
|
<option key={s} value={s}>
|
||||||
|
{s === 'all' ? 'Alle Severities' : s}
|
||||||
|
{s !== 'all' && sum.by_severity?.[s] != null ? ` (${sum.by_severity[s]})` : ''}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
<select value={status} onChange={e => setStatus(e.target.value)}
|
||||||
|
className="border border-gray-200 rounded px-2 py-1">
|
||||||
|
{STATUS_OPTS.map(s => (
|
||||||
|
<option key={s} value={s}>
|
||||||
|
{s === 'all' ? 'Alle Status' : STATUS_LABEL[s] ?? s}
|
||||||
|
{s !== 'all' && sum.by_status?.[s] != null ? ` (${sum.by_status[s]})` : ''}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
<select value={docType} onChange={e => setDocType(e.target.value)}
|
||||||
|
className="border border-gray-200 rounded px-2 py-1">
|
||||||
|
<option value="all">Alle Doc-Types</option>
|
||||||
|
{docTypes.map(d => (
|
||||||
|
<option key={d} value={d}>{d} ({sum.by_doc_type?.[d] ?? 0})</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
<input value={q} onChange={e => setQ(e.target.value)}
|
||||||
|
placeholder="Suche Label / Anbieter…"
|
||||||
|
className="border border-gray-200 rounded px-2 py-1 min-w-[180px]" />
|
||||||
|
<button onClick={csvExport}
|
||||||
|
className="ml-auto border border-gray-200 hover:border-gray-300 rounded px-2 py-1">
|
||||||
|
CSV exportieren
|
||||||
|
</button>
|
||||||
|
<span className="text-gray-500">{data.count} Treffer</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Findings table */}
|
||||||
|
<div className="border rounded-lg overflow-hidden">
|
||||||
|
<table className="w-full text-xs">
|
||||||
|
<thead className="bg-gray-50 text-gray-600">
|
||||||
|
<tr>
|
||||||
|
<th className="px-3 py-2 text-left">Quelle</th>
|
||||||
|
<th className="px-3 py-2 text-left">Doc</th>
|
||||||
|
<th className="px-3 py-2 text-left">Sev</th>
|
||||||
|
<th className="px-3 py-2 text-left">Status</th>
|
||||||
|
<th className="px-3 py-2 text-left">Finding</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{findings.map(f => (
|
||||||
|
<React.Fragment key={f.id}>
|
||||||
|
<tr className="border-t cursor-pointer hover:bg-gray-50"
|
||||||
|
onClick={() => setExpanded(expanded === f.id ? null : f.id)}>
|
||||||
|
<td className="px-3 py-2 text-gray-500 capitalize">{f.source_type}</td>
|
||||||
|
<td className="px-3 py-2 text-gray-700">{f.doc_type === '-' ? '—' : f.doc_type}</td>
|
||||||
|
<td className="px-3 py-2">
|
||||||
|
<span className={`px-2 py-0.5 rounded text-[10px] font-medium ${
|
||||||
|
SEVERITY_COLOR[f.severity] || 'bg-gray-100'
|
||||||
|
}`}>{f.severity}</span>
|
||||||
|
</td>
|
||||||
|
<td className="px-3 py-2 text-gray-600">{STATUS_LABEL[f.status] ?? f.status}</td>
|
||||||
|
<td className="px-3 py-2 text-gray-900">
|
||||||
|
{f.label}
|
||||||
|
{f.vendor_name && (
|
||||||
|
<span className="ml-2 text-[10px] text-gray-400">
|
||||||
|
· {f.vendor_name}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{f.payload?.risk_label && (
|
||||||
|
<span className={`ml-2 px-1.5 py-0.5 rounded text-[10px] font-medium ${
|
||||||
|
f.payload.risk_label === 'kritisch' ? 'bg-red-600 text-white' :
|
||||||
|
f.payload.risk_label === 'hoch' ? 'bg-red-100 text-red-800' :
|
||||||
|
f.payload.risk_label === 'mittel' ? 'bg-amber-100 text-amber-800' :
|
||||||
|
f.payload.risk_label === 'gering' ? 'bg-green-50 text-green-700' :
|
||||||
|
'bg-gray-100 text-gray-500'
|
||||||
|
}`}>Risk: {String(f.payload.risk_label)}</span>
|
||||||
|
)}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{expanded === f.id && (
|
||||||
|
<tr className="bg-gray-50/50">
|
||||||
|
<td colSpan={5} className="px-3 py-3 text-xs space-y-2">
|
||||||
|
{f.hint && (
|
||||||
|
<div className="text-gray-700">{f.hint}</div>
|
||||||
|
)}
|
||||||
|
{f.action_recipe?.fix_text && (
|
||||||
|
<div className="bg-amber-50 border-l-2 border-amber-300 pl-3 py-2">
|
||||||
|
<div className="font-medium text-amber-800 mb-1">Empfehlung</div>
|
||||||
|
<div className="whitespace-pre-line text-amber-900">
|
||||||
|
{f.action_recipe.fix_text}
|
||||||
|
</div>
|
||||||
|
{f.action_recipe.where && (
|
||||||
|
<div className="text-[10px] text-amber-700 mt-1">
|
||||||
|
Einfuegen in: {f.action_recipe.where}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{f.anchor_excerpt && (
|
||||||
|
<div className="bg-blue-50 border-l-2 border-blue-300 pl-3 py-2">
|
||||||
|
<div className="font-medium text-blue-800 mb-1">
|
||||||
|
Fundstelle im Dokument (Konfidenz {Math.round((f.anchor_conf || 0) * 100)}%)
|
||||||
|
</div>
|
||||||
|
<div className="italic text-blue-900">"{f.anchor_excerpt}"</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="text-[10px] text-gray-400">
|
||||||
|
Source: {f.source_type} · Regulation: {f.regulation || '—'}
|
||||||
|
{f.category && ` · Kategorie: ${f.category}`}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
)}
|
||||||
|
</React.Fragment>
|
||||||
|
))}
|
||||||
|
{findings.length === 0 && (
|
||||||
|
<tr><td colSpan={5} className="px-3 py-6 text-center text-gray-400">
|
||||||
|
Keine Findings fuer die aktuellen Filter.
|
||||||
|
</td></tr>
|
||||||
|
)}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import React, { useEffect, useState, useMemo } from 'react'
|
import React, { useEffect, useState, useMemo } from 'react'
|
||||||
import { use as useUnwrap } from 'react'
|
import { use as useUnwrap } from 'react'
|
||||||
|
import FindingsTab from './FindingsTab'
|
||||||
|
|
||||||
type MCRow = {
|
type MCRow = {
|
||||||
id: number
|
id: number
|
||||||
@@ -67,6 +68,7 @@ export default function AuditPage(
|
|||||||
const [filterReg, setFilterReg] = useState<string>('')
|
const [filterReg, setFilterReg] = useState<string>('')
|
||||||
const [filterDoc, setFilterDoc] = useState<string>('')
|
const [filterDoc, setFilterDoc] = useState<string>('')
|
||||||
const [expanded, setExpanded] = useState<number | null>(null)
|
const [expanded, setExpanded] = useState<number | null>(null)
|
||||||
|
const [tab, setTab] = useState<'mc' | 'all'>('all')
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
let cancelled = false
|
let cancelled = false
|
||||||
@@ -127,6 +129,25 @@ export default function AuditPage(
|
|||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Tab switcher */}
|
||||||
|
<div className="flex gap-2 border-b border-gray-200">
|
||||||
|
{([
|
||||||
|
{ key: 'all', label: 'Voll-Audit (alle Findings)' },
|
||||||
|
{ key: 'mc', label: 'Nur MC-Scorecard' },
|
||||||
|
] as const).map(t => (
|
||||||
|
<button key={t.key}
|
||||||
|
onClick={() => setTab(t.key)}
|
||||||
|
className={`px-4 py-2 text-sm border-b-2 -mb-px transition ${
|
||||||
|
tab === t.key
|
||||||
|
? 'border-blue-600 text-blue-700 font-medium'
|
||||||
|
: 'border-transparent text-gray-500 hover:text-gray-700'
|
||||||
|
}`}>{t.label}</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{tab === 'all' && <FindingsTab checkId={checkId} />}
|
||||||
|
|
||||||
|
{tab === 'mc' && <>
|
||||||
{/* Scorecard */}
|
{/* Scorecard */}
|
||||||
<div className="border rounded-lg overflow-hidden">
|
<div className="border rounded-lg overflow-hidden">
|
||||||
<div className="px-4 py-3 bg-blue-50 border-b border-blue-100">
|
<div className="px-4 py-3 bg-blue-50 border-b border-blue-100">
|
||||||
@@ -272,6 +293,7 @@ export default function AuditPage(
|
|||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
</>}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -166,6 +166,33 @@ async def _run_compliance_check(check_id: str, req: ComplianceCheckRequest):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# P7: TDM-Reservation-Check der Base-Domain (§ 44b UrhG).
|
||||||
|
# Bei reserved/denied: Run sofort beenden, kein Crawl.
|
||||||
|
try:
|
||||||
|
from compliance.services.tdm_reservation_check import (
|
||||||
|
check_tdm_reservation, is_crawl_allowed,
|
||||||
|
)
|
||||||
|
first_url = next(
|
||||||
|
(d.url for d in req.documents if d.url), "",
|
||||||
|
)
|
||||||
|
if first_url:
|
||||||
|
tdm = await check_tdm_reservation(first_url)
|
||||||
|
_compliance_check_jobs[check_id]["tdm"] = tdm
|
||||||
|
if not is_crawl_allowed(tdm):
|
||||||
|
_compliance_check_jobs[check_id]["status"] = "skipped_tdm"
|
||||||
|
_compliance_check_jobs[check_id]["error"] = (
|
||||||
|
f"TDM-Vorbehalt fuer {tdm.get('domain')} erkannt "
|
||||||
|
f"(status={tdm.get('status')}) — Crawl nach § 44b "
|
||||||
|
f"UrhG nicht zulaessig. Signals: "
|
||||||
|
f"{[s.get('src') for s in tdm.get('signals', [])]}"
|
||||||
|
)
|
||||||
|
_compliance_check_jobs[check_id]["progress_pct"] = 100
|
||||||
|
logger.info("TDM-skip check_id=%s domain=%s status=%s",
|
||||||
|
check_id, tdm.get("domain"), tdm.get("status"))
|
||||||
|
return
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("TDM-check failed (proceeding): %s", e)
|
||||||
|
|
||||||
# Step 1: Resolve texts (fetch from URL if needed) — 0-30%
|
# Step 1: Resolve texts (fetch from URL if needed) — 0-30%
|
||||||
_update(check_id, "Texte werden geladen...", 1)
|
_update(check_id, "Texte werden geladen...", 1)
|
||||||
doc_texts: dict[str, str] = {}
|
doc_texts: dict[str, str] = {}
|
||||||
@@ -526,15 +553,37 @@ async def _run_compliance_check(check_id: str, req: ComplianceCheckRequest):
|
|||||||
report_html = build_html_report(results, None, doc_texts)
|
report_html = build_html_report(results, None, doc_texts)
|
||||||
profile_html = _build_profile_html(profile)
|
profile_html = _build_profile_html(profile)
|
||||||
|
|
||||||
# O4: Vendor-Redundanz / EU-Alternativen + Cost-Savings-Block —
|
# O4: Vendor-Redundanz / EU-Alternativen + Cost-Savings-Block
|
||||||
# zwischen VVT und Doc-Report einsortiert, damit Geschaeftsfuehrung
|
|
||||||
# die Einsparung sieht bevor sie in die Detail-Pruefung geht.
|
|
||||||
from .agent_doc_check_redundancy import build_redundancy_html
|
from .agent_doc_check_redundancy import build_redundancy_html
|
||||||
redundancy_html = build_redundancy_html(redundancy_report)
|
redundancy_html = build_redundancy_html(redundancy_report)
|
||||||
|
|
||||||
|
# P1: Executive-Summary GANZ oben — CFO/GF sieht 4 KPIs + 2 CTAs.
|
||||||
|
from .agent_doc_check_exec_summary import build_exec_summary_html
|
||||||
|
# Site-Name fuer Header bestimmen (gleiche Logik wie Email-Subject)
|
||||||
|
url_company_for_exec = _company_name_from_url(doc_entries)
|
||||||
|
domain_for_exec = _extract_domain(doc_entries)
|
||||||
|
site_name_for_exec = url_company_for_exec or domain_for_exec or ""
|
||||||
|
exec_summary_html = build_exec_summary_html(
|
||||||
|
scorecard=scorecard,
|
||||||
|
previous_scorecard=prev_scorecard,
|
||||||
|
cmp_vendors=cmp_vendors,
|
||||||
|
redundancy_report=redundancy_report,
|
||||||
|
site_name=site_name_for_exec,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Reihenfolge — Sales-optimiert:
|
||||||
|
# 1) Exec-Summary (KPIs + Saving + CTAs)
|
||||||
|
# 2) summary_html (Konkrete Aufgaben fuer die Geschaeftsfuehrung)
|
||||||
|
# 3) scanned_urls (Quellen-Transparenz)
|
||||||
|
# 4) profile_html (Erkanntes Geschaeftsmodell)
|
||||||
|
# 5) scorecard_html (MC-Scorecard)
|
||||||
|
# 6) redundancy_html (Optimierungspotenzial — direkt nach Compliance-Score)
|
||||||
|
# 7) providers_html + vvt_html (Vendor-Liste)
|
||||||
|
# 8) report_html (Doc-Pruefung Details)
|
||||||
full_html = (
|
full_html = (
|
||||||
summary_html + scanned_html + profile_html + scorecard_html
|
exec_summary_html + summary_html + scanned_html + profile_html
|
||||||
+ providers_html + vvt_html + redundancy_html + report_html
|
+ scorecard_html + redundancy_html
|
||||||
|
+ providers_html + vvt_html + report_html
|
||||||
)
|
)
|
||||||
|
|
||||||
# Step 6: Send email — derive site name primarily from entered URL.
|
# Step 6: Send email — derive site name primarily from entered URL.
|
||||||
@@ -619,6 +668,21 @@ async def _run_compliance_check(check_id: str, req: ComplianceCheckRequest):
|
|||||||
vendors=cmp_vendors,
|
vendors=cmp_vendors,
|
||||||
profile=extracted_profile,
|
profile=extracted_profile,
|
||||||
)
|
)
|
||||||
|
# Unified findings (P5): bundle MC + Pflichtangaben + Vendor +
|
||||||
|
# Redundanz in one searchable table behind /agent/findings/<id>.
|
||||||
|
try:
|
||||||
|
from compliance.services.unified_findings_collector import collect
|
||||||
|
from compliance.services.unified_findings_store import record_findings
|
||||||
|
unified = collect(
|
||||||
|
check_id=check_id,
|
||||||
|
results=results,
|
||||||
|
cmp_vendors=cmp_vendors,
|
||||||
|
redundancy_report=redundancy_report,
|
||||||
|
doc_texts=doc_texts,
|
||||||
|
)
|
||||||
|
record_findings(check_id, unified)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Unified findings collect failed: %s", e)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Audit persistence skipped: %s", e)
|
logger.warning("Audit persistence skipped: %s", e)
|
||||||
|
|
||||||
@@ -696,11 +760,19 @@ async def _fetch_text(url: str, doc_type: str = "") -> tuple[str, list[dict]]:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Consent-tester fetch failed for %s: %s", url, e)
|
logger.warning("Consent-tester fetch failed for %s: %s", url, e)
|
||||||
|
|
||||||
# 2. Fallback: direct HTTP fetch (works for SSR pages like BMW)
|
# 2. Fallback: direct HTTP fetch (works for SSR pages like BMW).
|
||||||
|
# P7: kenntlicher UA + per-Domain Rate-Limit.
|
||||||
try:
|
try:
|
||||||
import re as _re
|
import re as _re
|
||||||
async with httpx.AsyncClient(timeout=30.0, follow_redirects=True) as client:
|
from compliance.services.compliance_user_agent import (
|
||||||
resp = await client.get(url)
|
default_request_headers, DomainRateLimiter,
|
||||||
|
)
|
||||||
|
async with httpx.AsyncClient(
|
||||||
|
timeout=30.0, follow_redirects=True,
|
||||||
|
headers=default_request_headers(),
|
||||||
|
) as client:
|
||||||
|
async with DomainRateLimiter(url):
|
||||||
|
resp = await client.get(url)
|
||||||
if resp.status_code == 200 and "text/html" in resp.headers.get("content-type", ""):
|
if resp.status_code == 200 and "text/html" in resp.headers.get("content-type", ""):
|
||||||
html = resp.text
|
html = resp.text
|
||||||
# Strip HTML tags, decode entities
|
# Strip HTML tags, decode entities
|
||||||
@@ -1135,8 +1207,25 @@ def _company_name_from_url(doc_entries: list[dict]) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _get_skip_types(profile) -> dict[str, str]:
|
def _get_skip_types(profile) -> dict[str, str]:
|
||||||
"""Doc_types to skip entirely. Currently empty — we check everything
|
"""Doc_types to skip entirely with a per-type reason message.
|
||||||
and flag irrelevant items as INFO instead of skipping."""
|
|
||||||
|
Heute primaer fuer OEM-Konfigurator-Pattern (BMW/Audi/Mercedes):
|
||||||
|
wenn die Site kein Direkt-Vertrieb macht, sind AGB/Widerruf/
|
||||||
|
Nutzungsbedingungen nicht Pflicht auf der Website — sie werden
|
||||||
|
beim Vertragshaendler ausgehaendigt.
|
||||||
|
"""
|
||||||
|
if getattr(profile, "no_direct_sales", False):
|
||||||
|
msg = (
|
||||||
|
"Nicht anwendbar — die Webseite schliesst keinen Direkt-"
|
||||||
|
"Kaufvertrag (OEM-Konfigurator-Pattern, Vertrag laeuft "
|
||||||
|
"ueber Vertragshaendler). AGB/Widerruf werden beim "
|
||||||
|
"Haendler ausgehaendigt."
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"agb": msg,
|
||||||
|
"widerruf": msg,
|
||||||
|
"nutzungsbedingungen": msg,
|
||||||
|
}
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
"""
|
||||||
|
Executive-Summary-Block — der oberste Email-Abschnitt.
|
||||||
|
|
||||||
|
Zeigt CFO / GF in 4 Zahlen den Gesamt-Mehrwert des Compliance-Checks:
|
||||||
|
1) Compliance-Score (Trend vs Vorlauf)
|
||||||
|
2) Anzahl analysierter Anbieter
|
||||||
|
3) Geschaetztes jaehrliches Sparpotenzial (Range)
|
||||||
|
4) Konsolidierungs-Potenzial (Anbieter koennen reduziert werden)
|
||||||
|
|
||||||
|
Plus zwei Big-CTA-Buttons:
|
||||||
|
- "Compliance-Maengel im Detail" → springt zum Doc-Pruefungs-Block
|
||||||
|
- "Konsolidierungs-Plan ansehen" → springt zum Redundanz-Block
|
||||||
|
|
||||||
|
Ziel: in 5 Sekunden sieht der Vorstand den ROI. Wenn neugierig, scrollt
|
||||||
|
er weiter in die Detail-Bloecke (die UNTER dieser Summary liegen).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_eur_range(low: int, high: int) -> str:
|
||||||
|
if not low and not high:
|
||||||
|
return "—"
|
||||||
|
if low == high:
|
||||||
|
return f"~{low:,} €".replace(",", ".")
|
||||||
|
return f"{low:,}–{high:,} €".replace(",", ".")
|
||||||
|
|
||||||
|
|
||||||
|
def build_exec_summary_html(
|
||||||
|
scorecard: dict | None,
|
||||||
|
previous_scorecard: dict | None,
|
||||||
|
cmp_vendors: list[dict] | None,
|
||||||
|
redundancy_report: dict | None,
|
||||||
|
site_name: str = "",
|
||||||
|
) -> str:
|
||||||
|
"""Build the top-of-email Executive Summary with 4 KPIs + 2 CTAs."""
|
||||||
|
# 1) Compliance-Score
|
||||||
|
pct = 0
|
||||||
|
delta_str = ""
|
||||||
|
score_color = "#94a3b8"
|
||||||
|
if scorecard:
|
||||||
|
totals = scorecard.get("totals") or {}
|
||||||
|
pct = int(totals.get("pct", 0))
|
||||||
|
score_color = ("#16a34a" if pct >= 80 else
|
||||||
|
"#d97706" if pct >= 50 else "#dc2626")
|
||||||
|
if previous_scorecard:
|
||||||
|
prev_pct = int((previous_scorecard.get("totals") or {}).get("pct", 0))
|
||||||
|
d = pct - prev_pct
|
||||||
|
if d:
|
||||||
|
trend_color = "#16a34a" if d > 0 else "#dc2626"
|
||||||
|
delta_str = (
|
||||||
|
f'<span style="font-size:14px;color:{trend_color};margin-left:6px">'
|
||||||
|
f'{"+" if d > 0 else ""}{d} pp</span>'
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2) Vendor-Count
|
||||||
|
n_vendors = len(cmp_vendors or [])
|
||||||
|
|
||||||
|
# 3+4) Saving + Konsolidierung
|
||||||
|
s = (redundancy_report or {}).get("summary") or {}
|
||||||
|
sav_low, sav_high = s.get("estimated_saving_year_eur", [0, 0])
|
||||||
|
n_consolidation = s.get("consolidation_potential", 0)
|
||||||
|
sav_pct = s.get("estimated_saving_pct", "—")
|
||||||
|
|
||||||
|
parts = [
|
||||||
|
'<div style="font-family:-apple-system,BlinkMacSystemFont,sans-serif;'
|
||||||
|
'max-width:700px;margin:0 auto 18px;padding:18px 22px;'
|
||||||
|
'background:linear-gradient(135deg,#1e293b 0%,#0f172a 100%);'
|
||||||
|
'border-radius:10px;color:white">',
|
||||||
|
|
||||||
|
f'<div style="font-size:11px;color:#94a3b8;text-transform:uppercase;'
|
||||||
|
f'letter-spacing:1.5px;margin-bottom:6px">Executive Summary</div>',
|
||||||
|
f'<h2 style="margin:0 0 16px;font-size:18px;color:white">'
|
||||||
|
f'Compliance-Check {site_name}</h2>',
|
||||||
|
|
||||||
|
# 2x2 KPI grid
|
||||||
|
'<table style="width:100%;border-collapse:separate;border-spacing:8px">',
|
||||||
|
|
||||||
|
# Row 1: Compliance + Vendor count
|
||||||
|
'<tr>',
|
||||||
|
f'<td style="width:50%;padding:12px 14px;background:rgba(255,255,255,0.05);'
|
||||||
|
f'border-radius:6px;border:1px solid rgba(255,255,255,0.08)">'
|
||||||
|
f'<div style="font-size:10px;color:#94a3b8;text-transform:uppercase;'
|
||||||
|
f'letter-spacing:1px;margin-bottom:4px">DSGVO / TDDDG / TMG Score</div>'
|
||||||
|
f'<div style="font-size:28px;font-weight:700;color:{score_color}">'
|
||||||
|
f'{pct}%{delta_str}</div>'
|
||||||
|
f'<div style="font-size:11px;color:#cbd5e1;margin-top:2px">'
|
||||||
|
f'aus {int((scorecard or {}).get("totals", {}).get("total", 0))} Pflicht-Pruefungen</div>'
|
||||||
|
f'</td>',
|
||||||
|
|
||||||
|
f'<td style="width:50%;padding:12px 14px;background:rgba(255,255,255,0.05);'
|
||||||
|
f'border-radius:6px;border:1px solid rgba(255,255,255,0.08)">'
|
||||||
|
f'<div style="font-size:10px;color:#94a3b8;text-transform:uppercase;'
|
||||||
|
f'letter-spacing:1px;margin-bottom:4px">Identifizierte Anbieter</div>'
|
||||||
|
f'<div style="font-size:28px;font-weight:700;color:white">{n_vendors}</div>'
|
||||||
|
f'<div style="font-size:11px;color:#cbd5e1;margin-top:2px">'
|
||||||
|
f'davon {n_consolidation} konsolidierbar</div>'
|
||||||
|
f'</td>',
|
||||||
|
'</tr>',
|
||||||
|
|
||||||
|
# Row 2: Saving + CTA-Hinweis
|
||||||
|
'<tr>',
|
||||||
|
f'<td colspan="2" style="padding:14px 16px;background:linear-gradient(90deg,'
|
||||||
|
f'rgba(16,185,129,0.15) 0%,rgba(16,185,129,0.05) 100%);'
|
||||||
|
f'border-radius:6px;border:1px solid rgba(16,185,129,0.3)">'
|
||||||
|
f'<div style="font-size:10px;color:#86efac;text-transform:uppercase;'
|
||||||
|
f'letter-spacing:1px;margin-bottom:4px">'
|
||||||
|
f'Geschaetztes Sparpotenzial pro Jahr (Tool-Lizenzen, ohne Media-Spend)</div>'
|
||||||
|
f'<div style="font-size:24px;font-weight:700;color:#34d399">'
|
||||||
|
f'{_fmt_eur_range(sav_low, sav_high)}'
|
||||||
|
f'<span style="font-size:14px;color:#86efac;margin-left:8px">({sav_pct})</span></div>'
|
||||||
|
f'<div style="font-size:11px;color:#cbd5e1;margin-top:4px">'
|
||||||
|
f'durch Konsolidierung redundanter Anbieter auf je 1 EU-Tool pro '
|
||||||
|
f'Funktions-Kategorie. <em>Schaetzbereich, mit dem Einkauf zu verifizieren.</em>'
|
||||||
|
f'</div></td>',
|
||||||
|
'</tr>',
|
||||||
|
|
||||||
|
'</table>',
|
||||||
|
|
||||||
|
# CTAs
|
||||||
|
'<div style="margin-top:14px;padding-top:12px;border-top:1px solid '
|
||||||
|
'rgba(255,255,255,0.1);text-align:center">',
|
||||||
|
'<a href="#mc-scorecard" style="display:inline-block;padding:8px 16px;'
|
||||||
|
'background:#7c3aed;color:white;text-decoration:none;border-radius:6px;'
|
||||||
|
'font-size:12px;font-weight:600;margin-right:8px">'
|
||||||
|
'Compliance-Maengel im Detail →</a>',
|
||||||
|
'<a href="#optimierungspotenzial" style="display:inline-block;padding:8px 16px;'
|
||||||
|
'background:#10b981;color:white;text-decoration:none;border-radius:6px;'
|
||||||
|
'font-size:12px;font-weight:600">'
|
||||||
|
'Konsolidierungs-Plan →</a>',
|
||||||
|
'</div>',
|
||||||
|
|
||||||
|
'</div>',
|
||||||
|
]
|
||||||
|
return "".join(parts)
|
||||||
@@ -421,10 +421,18 @@ def _render_vendor_row_full(v: dict) -> str:
|
|||||||
f'{", ".join(flags[:4])}</div>'
|
f'{", ".join(flags[:4])}</div>'
|
||||||
f'{actions_html}'
|
f'{actions_html}'
|
||||||
)
|
)
|
||||||
|
risk = v.get("compliance_risk") or {}
|
||||||
|
risk_label = risk.get("label") or ""
|
||||||
|
risk_badge = ""
|
||||||
|
if risk_label and risk_label != "unklar":
|
||||||
|
rc = {"kritisch": ("#dc2626", "#fff"), "hoch": ("#fecaca", "#991b1b"),
|
||||||
|
"mittel": ("#fde68a", "#92400e"), "gering": ("#d1fae5", "#065f46")}.get(risk_label, ("#e5e7eb", "#475569"))
|
||||||
|
risk_badge = (f'<span style="margin-left:6px;padding:1px 5px;border-radius:3px;font-size:9px;'
|
||||||
|
f'background:{rc[0]};color:{rc[1]}">Risk: {risk_label}</span>')
|
||||||
return (
|
return (
|
||||||
f'<tr style="border-top:1px solid #e2e8f0">'
|
f'<tr style="border-top:1px solid #e2e8f0">'
|
||||||
f'<td style="padding:6px 8px;color:#1e293b;font-size:11px">'
|
f'<td style="padding:6px 8px;color:#1e293b;font-size:11px">'
|
||||||
f'{name}{flag_str}</td>'
|
f'{name}{risk_badge}{flag_str}</td>'
|
||||||
f'<td style="padding:6px 8px;color:#475569;font-size:11px">{category}</td>'
|
f'<td style="padding:6px 8px;color:#475569;font-size:11px">{category}</td>'
|
||||||
f'<td style="padding:6px 8px;color:#475569;font-size:11px">{country}</td>'
|
f'<td style="padding:6px 8px;color:#475569;font-size:11px">{country}</td>'
|
||||||
f'<td style="padding:6px 8px;text-align:center;color:#475569;font-size:11px">'
|
f'<td style="padding:6px 8px;text-align:center;color:#475569;font-size:11px">'
|
||||||
|
|||||||
@@ -28,9 +28,10 @@ def build_redundancy_html(report: dict | None) -> str:
|
|||||||
pct = s.get("estimated_saving_pct") or "n/a"
|
pct = s.get("estimated_saving_pct") or "n/a"
|
||||||
|
|
||||||
parts = [
|
parts = [
|
||||||
'<div style="font-family:-apple-system,BlinkMacSystemFont,sans-serif;'
|
'<div id="optimierungspotenzial" style="font-family:-apple-system,'
|
||||||
'max-width:700px;margin:0 auto 16px;padding:14px 18px;'
|
'BlinkMacSystemFont,sans-serif;max-width:700px;margin:0 auto 16px;'
|
||||||
'background:#fef3c7;border:1px solid #fcd34d;border-radius:8px">',
|
'padding:14px 18px;background:#fef3c7;border:1px solid #fcd34d;'
|
||||||
|
'border-radius:8px">',
|
||||||
'<h3 style="margin:0 0 6px;font-size:14px;color:#92400e">'
|
'<h3 style="margin:0 0 6px;font-size:14px;color:#92400e">'
|
||||||
'Optimierungspotenzial: Redundanzen + EU-Alternativen</h3>',
|
'Optimierungspotenzial: Redundanzen + EU-Alternativen</h3>',
|
||||||
f'<p style="margin:0 0 10px;font-size:11px;color:#78350f">'
|
f'<p style="margin:0 0 10px;font-size:11px;color:#78350f">'
|
||||||
|
|||||||
@@ -134,7 +134,9 @@ def build_management_summary(results: list[DocCheckResult]) -> str:
|
|||||||
ok = [r for r in results if r.completeness_pct == 100 and not r.error]
|
ok = [r for r in results if r.completeness_pct == 100 and not r.error]
|
||||||
fixable = [r for r in results if 0 < r.completeness_pct < 100 and not r.error]
|
fixable = [r for r in results if 0 < r.completeness_pct < 100 and not r.error]
|
||||||
critical = [r for r in results if r.completeness_pct == 0 and not r.error]
|
critical = [r for r in results if r.completeness_pct == 0 and not r.error]
|
||||||
errors = [r for r in results if r.error]
|
not_applicable = [r for r in results if r.error
|
||||||
|
and r.error.startswith("Nicht anwendbar")]
|
||||||
|
errors = [r for r in results if r.error and r not in not_applicable]
|
||||||
|
|
||||||
html = [
|
html = [
|
||||||
'<div style="font-family:-apple-system,BlinkMacSystemFont,sans-serif;'
|
'<div style="font-family:-apple-system,BlinkMacSystemFont,sans-serif;'
|
||||||
@@ -150,17 +152,24 @@ def build_management_summary(results: list[DocCheckResult]) -> str:
|
|||||||
html.append('<p>Keine Dokumente geprueft.</p></div>')
|
html.append('<p>Keine Dokumente geprueft.</p></div>')
|
||||||
return "\n".join(html)
|
return "\n".join(html)
|
||||||
|
|
||||||
|
na_note = (
|
||||||
|
f' Zusaetzlich {len(not_applicable)} Dokument{"" if len(not_applicable) == 1 else "e"} '
|
||||||
|
f'als NICHT ANWENDBAR markiert (kein Direkt-Vertrieb — '
|
||||||
|
f'OEM-Konfigurator-Pattern).' if not_applicable else ""
|
||||||
|
)
|
||||||
if len(ok) == total:
|
if len(ok) == total:
|
||||||
html.append(
|
html.append(
|
||||||
'<p style="color:#16a34a;font-weight:600;font-size:15px">'
|
f'<p style="color:#16a34a;font-weight:600;font-size:15px">'
|
||||||
'Alle Dokumente sind vollstaendig. Keine dringenden Massnahmen noetig.</p>'
|
f'Alle Dokumente sind vollstaendig. Keine dringenden Massnahmen noetig.'
|
||||||
|
f'{na_note}</p>'
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
html.append(
|
html.append(
|
||||||
f'<p style="font-size:14px;color:#475569">'
|
f'<p style="font-size:14px;color:#475569">'
|
||||||
f'{len(ok)} von {total} Dokumenten sind vollstaendig. '
|
f'{len(ok)} von {total} Dokumenten sind vollstaendig. '
|
||||||
f'{len(fixable)} brauchen Korrekturen'
|
f'{len(fixable)} brauchen Korrekturen'
|
||||||
f'{f", {len(critical)} fehlen oder sind unbrauchbar" if critical else ""}.</p>'
|
f'{f", {len(critical)} fehlen oder sind unbrauchbar" if critical else ""}.'
|
||||||
|
f'{na_note}</p>'
|
||||||
)
|
)
|
||||||
|
|
||||||
# Concrete actions
|
# Concrete actions
|
||||||
@@ -279,10 +288,13 @@ def _render_document(html: list[str], r: DocCheckResult, doc_text: str = "") ->
|
|||||||
r.error.startswith("Nicht eingereicht")
|
r.error.startswith("Nicht eingereicht")
|
||||||
or r.error.startswith("Auf der Website nicht gefunden")
|
or r.error.startswith("Auf der Website nicht gefunden")
|
||||||
)
|
)
|
||||||
|
is_not_applicable = bool(r.error) and r.error.startswith("Nicht anwendbar")
|
||||||
if is_missing:
|
if is_missing:
|
||||||
status_label = ("NICHT GEFUNDEN"
|
status_label = ("NICHT GEFUNDEN"
|
||||||
if r.error.startswith("Auf der Website")
|
if r.error.startswith("Auf der Website")
|
||||||
else "NICHT EINGEREICHT")
|
else "NICHT EINGEREICHT")
|
||||||
|
elif is_not_applicable:
|
||||||
|
status_label = "NICHT ANWENDBAR"
|
||||||
elif r.error:
|
elif r.error:
|
||||||
status_label = "FEHLER"
|
status_label = "FEHLER"
|
||||||
|
|
||||||
@@ -330,6 +342,13 @@ def _render_document(html: list[str], r: DocCheckResult, doc_text: str = "") ->
|
|||||||
'background:#fafafa;border-top:1px solid #f3f4f6">'
|
'background:#fafafa;border-top:1px solid #f3f4f6">'
|
||||||
+ body_msg + '</div>'
|
+ body_msg + '</div>'
|
||||||
)
|
)
|
||||||
|
elif is_not_applicable:
|
||||||
|
html.append(
|
||||||
|
'<div style="padding:12px 16px;color:#475569;font-size:12px;'
|
||||||
|
'background:#f1f5f9;border-top:1px solid #cbd5e1;border-left:'
|
||||||
|
'3px solid #94a3b8">'
|
||||||
|
+ r.error + '</div>'
|
||||||
|
)
|
||||||
elif r.error:
|
elif r.error:
|
||||||
html.append(f'<div style="padding:12px 16px;color:#991b1b">{r.error}</div>')
|
html.append(f'<div style="padding:12px 16px;color:#991b1b">{r.error}</div>')
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ def build_scorecard_html(
|
|||||||
trend_str = _delta_badge(overall_pct, prev_total_pct) if prev_total_pct is not None else ""
|
trend_str = _delta_badge(overall_pct, prev_total_pct) if prev_total_pct is not None else ""
|
||||||
|
|
||||||
head = (
|
head = (
|
||||||
'<div style="font-family:-apple-system,BlinkMacSystemFont,sans-serif;'
|
'<div id="mc-scorecard" style="font-family:-apple-system,BlinkMacSystemFont,sans-serif;'
|
||||||
'max-width:700px;margin:0 auto 16px;padding:12px 16px;'
|
'max-width:700px;margin:0 auto 16px;padding:12px 16px;'
|
||||||
'background:#f0f9ff;border:1px solid #bae6fd;border-radius:8px">'
|
'background:#f0f9ff;border:1px solid #bae6fd;border-radius:8px">'
|
||||||
'<h3 style="margin:0 0 6px;font-size:14px;color:#0369a1">'
|
'<h3 style="margin:0 0 6px;font-size:14px;color:#0369a1">'
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
"""
|
||||||
|
Voll-Audit Findings Router — unified view across all 4 finding sources.
|
||||||
|
|
||||||
|
Endpoint:
|
||||||
|
GET /api/compliance/agent/findings/{check_id}
|
||||||
|
?source=mc|pflichtangabe|vendor|redundanz|all
|
||||||
|
&severity=CRITICAL|HIGH|MEDIUM|LOW|INFO|all
|
||||||
|
&doc_type=impressum|dse|cookie|...|all
|
||||||
|
&status=failed|passed|skipped|na|info|all
|
||||||
|
&q=<freitext>
|
||||||
|
&limit=<int>
|
||||||
|
|
||||||
|
Liefert summary + filtered findings list. Frontend rendert daraus den
|
||||||
|
Voll-Audit-Tab unter /sdk/agent/audit/<check_id>.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from fastapi import APIRouter, HTTPException, Query
|
||||||
|
|
||||||
|
from compliance.services.unified_findings_store import (
|
||||||
|
findings_summary,
|
||||||
|
list_findings,
|
||||||
|
)
|
||||||
|
from compliance.services.compliance_audit_log import get_check_run
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/compliance/agent", tags=["agent"])
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_domain(d: str) -> str:
|
||||||
|
if not d:
|
||||||
|
return ""
|
||||||
|
if "://" not in d:
|
||||||
|
d = "https://" + d
|
||||||
|
host = urlparse(d).netloc.lower()
|
||||||
|
return host[4:] if host.startswith("www.") else host
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/findings/{check_id}")
|
||||||
|
def get_findings(
|
||||||
|
check_id: str,
|
||||||
|
source: str | None = Query(None, description="mc|pflichtangabe|vendor|redundanz|all"),
|
||||||
|
severity: str | None = Query(None, description="CRITICAL|HIGH|MEDIUM|LOW|INFO|all"),
|
||||||
|
doc_type: str | None = Query(None),
|
||||||
|
status: str | None = Query(None, description="failed|passed|skipped|na|info|all"),
|
||||||
|
q: str | None = Query(None, description="freitext-suche label/vendor"),
|
||||||
|
limit: int = Query(1000, ge=1, le=5000),
|
||||||
|
expected_domain: str | None = Query(
|
||||||
|
None, description="Hard-Assertion: Run muss zu dieser Domain gehoeren (Cross-Tenant-Schutz)",
|
||||||
|
),
|
||||||
|
) -> dict:
|
||||||
|
"""Return aggregated findings + summary counters for a check run."""
|
||||||
|
# P7-Restpunkt: optionale Domain-Assertion. Verhindert dass ein Frontend
|
||||||
|
# einen check_id einer fremden Tenant-Domain anfragen kann.
|
||||||
|
if expected_domain:
|
||||||
|
run = get_check_run(check_id)
|
||||||
|
actual = _normalize_domain((run or {}).get("base_domain") or "")
|
||||||
|
if not run or actual != _normalize_domain(expected_domain):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=403,
|
||||||
|
detail=f"Cross-tenant access blocked: check_id {check_id} "
|
||||||
|
f"gehoert zu Domain '{actual or '?'}', angefragt: "
|
||||||
|
f"'{_normalize_domain(expected_domain)}'",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
summary = findings_summary(check_id)
|
||||||
|
findings = list_findings(
|
||||||
|
check_id=check_id,
|
||||||
|
source_type=source,
|
||||||
|
severity=severity,
|
||||||
|
doc_type=doc_type,
|
||||||
|
status=status,
|
||||||
|
q=q,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"found": summary.get("total", 0) > 0,
|
||||||
|
"check_id": check_id,
|
||||||
|
"summary": summary,
|
||||||
|
"filter": {
|
||||||
|
"source": source or "all",
|
||||||
|
"severity": severity or "all",
|
||||||
|
"doc_type": doc_type or "all",
|
||||||
|
"status": status or "all",
|
||||||
|
"q": q or "",
|
||||||
|
"limit": limit,
|
||||||
|
},
|
||||||
|
"count": len(findings),
|
||||||
|
"findings": findings,
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("get_findings failed for %s", check_id)
|
||||||
|
return {
|
||||||
|
"found": False,
|
||||||
|
"check_id": check_id,
|
||||||
|
"error": str(e)[:200],
|
||||||
|
"summary": {},
|
||||||
|
"count": 0,
|
||||||
|
"findings": [],
|
||||||
|
}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
"""
|
||||||
|
Saving-Scan-Funnel Endpoint — Marketing-Lead → Compliance-Check.
|
||||||
|
|
||||||
|
Externes Form (https://breakpilot.ai/savings-scan) postet hier:
|
||||||
|
POST /api/compliance/agent/saving-scan/start
|
||||||
|
Body: {"url": "...", "email": "..."}
|
||||||
|
|
||||||
|
Server-side:
|
||||||
|
1. Validierung URL + Email (E-Mail-Regex, URL-Schema).
|
||||||
|
2. Rate-Limit: max 1 vollstaendiger Scan / Domain / 24h
|
||||||
|
(saving_scan_allowed aus compliance_user_agent).
|
||||||
|
3. Lead persistieren (saving_scan_leads in Sidecar-SQLite) — fuer
|
||||||
|
spaeteren Report-Versand + Sales-Follow-Up.
|
||||||
|
4. Compliance-Check starten mit Auto-Discovery (DocumentInput leer
|
||||||
|
ausser Homepage). Der bestehende Worker laeuft TDM-Check, dann
|
||||||
|
Discovery, dann Pruefung.
|
||||||
|
5. check_id zurueck — Frontend pollt /compliance-check/<check_id>.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import uuid as _uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from fastapi import APIRouter, HTTPException
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from compliance.services.compliance_user_agent import (
|
||||||
|
base_domain_of, saving_scan_allowed,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/compliance/agent", tags=["agent"])
|
||||||
|
|
||||||
|
DB_PATH = os.getenv("COMPLIANCE_AUDIT_DB", "/data/compliance_audits.db")
|
||||||
|
|
||||||
|
_EMAIL_RE = re.compile(r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$")
|
||||||
|
_URL_RE = re.compile(r"^https?://[A-Za-z0-9.-]+(/.*)?$")
|
||||||
|
|
||||||
|
|
||||||
|
class SavingScanRequest(BaseModel):
|
||||||
|
url: str = Field(..., min_length=4, max_length=400)
|
||||||
|
email: str = Field(..., min_length=5, max_length=200)
|
||||||
|
consent: bool = Field(
|
||||||
|
True, description="Marketing-Consent fuer Sales-Follow-Up — "
|
||||||
|
"muss True sein laut Form-Checkbox.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SavingScanResponse(BaseModel):
|
||||||
|
check_id: str
|
||||||
|
status: str
|
||||||
|
message: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_leads_table() -> None:
|
||||||
|
Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.executescript("""
|
||||||
|
CREATE TABLE IF NOT EXISTS saving_scan_leads (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
ts TEXT NOT NULL,
|
||||||
|
email TEXT NOT NULL,
|
||||||
|
url TEXT NOT NULL,
|
||||||
|
base_domain TEXT NOT NULL,
|
||||||
|
check_id TEXT,
|
||||||
|
consent INTEGER NOT NULL,
|
||||||
|
source TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_leads_domain ON saving_scan_leads(base_domain, ts);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_leads_email ON saving_scan_leads(email, ts);
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def _persist_lead(email: str, url: str, check_id: str, consent: bool) -> None:
|
||||||
|
try:
|
||||||
|
_ensure_leads_table()
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO saving_scan_leads "
|
||||||
|
"(ts, email, url, base_domain, check_id, consent, source) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
(
|
||||||
|
datetime.now(timezone.utc).isoformat(),
|
||||||
|
email.lower().strip(),
|
||||||
|
url,
|
||||||
|
base_domain_of(url),
|
||||||
|
check_id,
|
||||||
|
1 if consent else 0,
|
||||||
|
"saving_scan_form",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("persist lead failed: %s", e)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_url(url: str) -> str:
|
||||||
|
"""Strip path → behaupt nur Homepage, der Discover findet den Rest."""
|
||||||
|
if "://" not in url:
|
||||||
|
url = "https://" + url
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
p = urlparse(url)
|
||||||
|
return f"{p.scheme}://{p.netloc}/"
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/saving-scan/start", response_model=SavingScanResponse)
|
||||||
|
async def start_saving_scan(req: SavingScanRequest) -> SavingScanResponse:
|
||||||
|
"""Trigger compliance check from the marketing-funnel form."""
|
||||||
|
if not _EMAIL_RE.match(req.email):
|
||||||
|
raise HTTPException(400, "Ungueltige E-Mail-Adresse.")
|
||||||
|
if not _URL_RE.match(req.url):
|
||||||
|
raise HTTPException(400, "URL muss mit http:// oder https:// beginnen.")
|
||||||
|
if not req.consent:
|
||||||
|
raise HTTPException(400, "Marketing-Consent erforderlich.")
|
||||||
|
|
||||||
|
domain = base_domain_of(req.url)
|
||||||
|
if not domain:
|
||||||
|
raise HTTPException(400, "Konnte Domain nicht ermitteln.")
|
||||||
|
|
||||||
|
allowed, wait_s = saving_scan_allowed(req.url)
|
||||||
|
if not allowed:
|
||||||
|
raise HTTPException(
|
||||||
|
429,
|
||||||
|
f"Fuer '{domain}' wurde in den letzten 24h bereits ein Scan "
|
||||||
|
f"durchgefuehrt. Bitte in {wait_s // 3600}h {wait_s % 3600 // 60}min "
|
||||||
|
f"erneut versuchen.",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Lazy import to avoid circular dependency at module load.
|
||||||
|
from compliance.api.agent_compliance_check_routes import (
|
||||||
|
DocumentInput,
|
||||||
|
ComplianceCheckRequest,
|
||||||
|
_run_compliance_check,
|
||||||
|
_compliance_check_jobs,
|
||||||
|
)
|
||||||
|
|
||||||
|
homepage = _normalize_url(req.url)
|
||||||
|
check_id = str(_uuid.uuid4())[:8]
|
||||||
|
_compliance_check_jobs[check_id] = {
|
||||||
|
"status": "running",
|
||||||
|
"progress": "Saving-Scan gestartet — Auto-Discovery laeuft...",
|
||||||
|
"progress_pct": 0,
|
||||||
|
"result": None,
|
||||||
|
"error": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Single "other" entry forces auto-discovery to fill in the rest.
|
||||||
|
docs = [DocumentInput(doc_type="other", url=homepage)]
|
||||||
|
check_req = ComplianceCheckRequest(
|
||||||
|
documents=docs, recipient=req.email.lower().strip(),
|
||||||
|
)
|
||||||
|
|
||||||
|
_persist_lead(req.email, req.url, check_id, req.consent)
|
||||||
|
asyncio.create_task(_run_compliance_check(check_id, check_req))
|
||||||
|
|
||||||
|
logger.info("saving-scan start: check_id=%s domain=%s email=%s",
|
||||||
|
check_id, domain, req.email[:3] + "***")
|
||||||
|
return SavingScanResponse(
|
||||||
|
check_id=check_id,
|
||||||
|
status="running",
|
||||||
|
message=f"Scan gestartet fuer {domain}. Bericht in ~3-5 Minuten.",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/saving-scan/lead-count")
|
||||||
|
def saving_scan_lead_count() -> dict:
|
||||||
|
"""Diagnostik fuer das Sales-Dashboard."""
|
||||||
|
try:
|
||||||
|
_ensure_leads_table()
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
total = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM saving_scan_leads",
|
||||||
|
).fetchone()[0]
|
||||||
|
last_24h = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM saving_scan_leads "
|
||||||
|
"WHERE ts > datetime('now', '-1 day')",
|
||||||
|
).fetchone()[0]
|
||||||
|
top_domains = conn.execute(
|
||||||
|
"SELECT base_domain, COUNT(*) AS n FROM saving_scan_leads "
|
||||||
|
"GROUP BY base_domain ORDER BY n DESC LIMIT 10",
|
||||||
|
).fetchall()
|
||||||
|
return {
|
||||||
|
"total_leads": total,
|
||||||
|
"last_24h": last_24h,
|
||||||
|
"top_domains": [{"domain": d, "scans": n} for d, n in top_domains],
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
return {"error": str(e)[:200]}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
"""
|
||||||
|
k-Anonymitaets-Helper fuer Branchen-Benchmarks (P6-Vorbereitung).
|
||||||
|
|
||||||
|
Vor jeder Veroeffentlichung von Benchmark-Aussagen pruefen, ob die
|
||||||
|
zugrundeliegende Stichprobe gross genug ist, dass keine Re-Identifikation
|
||||||
|
einzelner Hersteller moeglich wird.
|
||||||
|
|
||||||
|
Default k=5: jede publizierbare Aussage muss auf mindestens 5 verschiedenen
|
||||||
|
Datensubjekten (z.B. OEM-Sites) beruhen. Bei OEM-Markt mit ~30 Spielern
|
||||||
|
ist k=5 das Minimum, um "ein deutscher Premium-Hersteller mit X Modellen"
|
||||||
|
auszuschliessen.
|
||||||
|
|
||||||
|
Memory: feedback_oem_data_legal.md + project_legal_contracts_2026_07.md.
|
||||||
|
|
||||||
|
Verwendung:
|
||||||
|
from compliance.services.benchmark_k_anonymity import (
|
||||||
|
enforce_k_anonymity, quantize_value, KAnonymityError,
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = [...] # pro Hersteller 1 Row
|
||||||
|
safe_groups = enforce_k_anonymity(rows, group_keys=["segment", "country"])
|
||||||
|
# safe_groups: nur Gruppen mit count >= 5 zurueck
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
DEFAULT_K = 5
|
||||||
|
|
||||||
|
|
||||||
|
class KAnonymityError(RuntimeError):
|
||||||
|
"""Stichprobe ist zu klein fuer eine publizierbare Aussage."""
|
||||||
|
|
||||||
|
|
||||||
|
def assert_min_sample(n: int, k: int = DEFAULT_K, context: str = "") -> None:
|
||||||
|
"""Wirft KAnonymityError wenn n < k."""
|
||||||
|
if n < k:
|
||||||
|
raise KAnonymityError(
|
||||||
|
f"Stichprobe zu klein fuer Publikation: n={n} < k={k}"
|
||||||
|
+ (f" — Kontext: {context}" if context else "")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def quantize_value(value: float | int, step: int = 5) -> int:
|
||||||
|
"""Quantisiere Zahlenwerte auf step-Vielfache (Generalisierung).
|
||||||
|
|
||||||
|
quantize_value(67, 5) -> 65
|
||||||
|
quantize_value(83, 10) -> 80
|
||||||
|
|
||||||
|
Verhindert exakte Identifizierung ueber numerische Signale.
|
||||||
|
"""
|
||||||
|
if step <= 0:
|
||||||
|
return int(value)
|
||||||
|
return int(value // step) * step
|
||||||
|
|
||||||
|
|
||||||
|
def quantize_range(value: float | int, step: int = 10) -> str:
|
||||||
|
"""Gib ein Range-Bucket zurueck als String: '60-70%', '80-90%'."""
|
||||||
|
base = quantize_value(value, step)
|
||||||
|
return f"{base}-{base + step}%"
|
||||||
|
|
||||||
|
|
||||||
|
def group_and_count(
|
||||||
|
rows: Iterable[dict],
|
||||||
|
keys: list[str],
|
||||||
|
) -> dict[tuple, int]:
|
||||||
|
"""Gruppiere Rows nach allen `keys` und zaehle pro Bucket."""
|
||||||
|
counts: dict[tuple, int] = {}
|
||||||
|
for r in rows:
|
||||||
|
bucket = tuple(r.get(k, "") for k in keys)
|
||||||
|
counts[bucket] = counts.get(bucket, 0) + 1
|
||||||
|
return counts
|
||||||
|
|
||||||
|
|
||||||
|
def enforce_k_anonymity(
|
||||||
|
rows: list[dict],
|
||||||
|
group_keys: list[str],
|
||||||
|
k: int = DEFAULT_K,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Filtere Rows so, dass jede ueberlebende Gruppe >= k Mitglieder hat.
|
||||||
|
|
||||||
|
Returns: Rows die in ausreichend grossen Gruppen sind.
|
||||||
|
Rows in zu kleinen Gruppen werden suppressed (entfernt).
|
||||||
|
"""
|
||||||
|
counts = group_and_count(rows, group_keys)
|
||||||
|
safe_buckets = {bucket for bucket, n in counts.items() if n >= k}
|
||||||
|
return [
|
||||||
|
r for r in rows
|
||||||
|
if tuple(r.get(key, "") for key in group_keys) in safe_buckets
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def summarize_benchmark(
|
||||||
|
rows: list[dict],
|
||||||
|
group_keys: list[str],
|
||||||
|
measure_key: str,
|
||||||
|
k: int = DEFAULT_K,
|
||||||
|
quantize_step: int = 5,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Erzeuge publizierbare Benchmark-Aggregat-Zeilen.
|
||||||
|
|
||||||
|
Pro Gruppe: count, mean (quantisiert), only-if count >= k.
|
||||||
|
Liefert sortiert nach count desc.
|
||||||
|
|
||||||
|
Beispiel:
|
||||||
|
rows = [{"segment": "premium", "consent_score": 84}, ...]
|
||||||
|
summarize_benchmark(rows, ["segment"], "consent_score")
|
||||||
|
-> [{"segment": "premium", "n": 8, "mean_quantized": 80}, ...]
|
||||||
|
"""
|
||||||
|
buckets: dict[tuple, list[float]] = {}
|
||||||
|
for r in rows:
|
||||||
|
bucket = tuple(r.get(k, "") for k in group_keys)
|
||||||
|
val = r.get(measure_key)
|
||||||
|
if val is not None:
|
||||||
|
buckets.setdefault(bucket, []).append(float(val))
|
||||||
|
|
||||||
|
out: list[dict] = []
|
||||||
|
for bucket, values in buckets.items():
|
||||||
|
n = len(values)
|
||||||
|
if n < k:
|
||||||
|
continue
|
||||||
|
mean = sum(values) / n
|
||||||
|
entry: dict[str, Any] = {key: bucket[i] for i, key in enumerate(group_keys)}
|
||||||
|
entry["n"] = n
|
||||||
|
entry["mean_quantized"] = quantize_value(mean, quantize_step)
|
||||||
|
entry["mean_range"] = quantize_range(mean, quantize_step * 2)
|
||||||
|
out.append(entry)
|
||||||
|
out.sort(key=lambda e: e["n"], reverse=True)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def safe_to_publish(
|
||||||
|
statement: str,
|
||||||
|
sample_size: int,
|
||||||
|
k: int = DEFAULT_K,
|
||||||
|
) -> tuple[bool, str]:
|
||||||
|
"""Validator fuer Marketing/Press-Statements.
|
||||||
|
|
||||||
|
Returns (ok, message). Wenn ok=False, NICHT publishen.
|
||||||
|
"""
|
||||||
|
if sample_size < k:
|
||||||
|
return False, (
|
||||||
|
f'Aussage NICHT publizierbar: "{statement[:60]}…" '
|
||||||
|
f'(n={sample_size} < k={k}). Risiko: Re-Identifikation '
|
||||||
|
f'einzelner Hersteller moeglich.'
|
||||||
|
)
|
||||||
|
return True, f"OK (n={sample_size}, k={k})"
|
||||||
@@ -28,6 +28,12 @@ class BusinessProfile:
|
|||||||
needs_odr: bool = False # Online-Streitbeilegung
|
needs_odr: bool = False # Online-Streitbeilegung
|
||||||
detected_services: list[str] = field(default_factory=list)
|
detected_services: list[str] = field(default_factory=list)
|
||||||
confidence: float = 0.0
|
confidence: float = 0.0
|
||||||
|
# Wenn True: die Site selbst schliesst KEINEN Direktkauf-Vertrag
|
||||||
|
# (typisch OEM-Konfigurator-Sites BMW/Audi/Mercedes — Vertrag laeuft
|
||||||
|
# ueber den Vertragshaendler, nicht die Hersteller-Webseite).
|
||||||
|
# Konsequenz: AGB/Widerruf/Nutzungsbedingungen sind NICHT PFLICHT
|
||||||
|
# auf der Website, sondern werden beim Haendler ausgehaendigt.
|
||||||
|
no_direct_sales: bool = False
|
||||||
|
|
||||||
|
|
||||||
# ── Keyword lists ────────────────────────────────────────────────────
|
# ── Keyword lists ────────────────────────────────────────────────────
|
||||||
@@ -319,4 +325,49 @@ async def detect_business_profile(documents: dict[str, str]) -> BusinessProfile:
|
|||||||
"steuerberater": "finance", "architekt": "craft"}
|
"steuerberater": "finance", "architekt": "craft"}
|
||||||
profile.industry = prof_map.get(profile.regulated_profession_type, "unknown")
|
profile.industry = prof_map.get(profile.regulated_profession_type, "unknown")
|
||||||
|
|
||||||
|
# ── no_direct_sales (OEM-Konfigurator-Pattern) ───────────────
|
||||||
|
# Hersteller-Sites die nur konfigurieren + zu Vertragshaendlern
|
||||||
|
# weiterleiten (BMW/Audi/Mercedes/VW/Porsche) schliessen KEINEN
|
||||||
|
# Direkt-Kaufvertrag. AGB/Widerruf/Nutzungsbedingungen sind dort
|
||||||
|
# nicht Pflicht — werden beim Haendler ausgehaendigt.
|
||||||
|
profile.no_direct_sales = _detect_no_direct_sales(full_text)
|
||||||
|
|
||||||
return profile
|
return profile
|
||||||
|
|
||||||
|
|
||||||
|
# Indikatoren: Site verweist primaer auf Vertragshaendler/Niederlassungen
|
||||||
|
# statt einen eigenen Checkout-Vertragsabschluss zu bieten.
|
||||||
|
_NO_DIRECT_SALES_POSITIVE = [
|
||||||
|
"vertragshaendler", "vertragshändler", "vertragspartner",
|
||||||
|
"vertragswerkstatt", "haendlersuche", "händlersuche",
|
||||||
|
"niederlassung", "vertretung", "autorisierter haendler",
|
||||||
|
"autorisierter händler", "ihr haendler vor ort",
|
||||||
|
"ihr händler vor ort", "haendler in ihrer naehe",
|
||||||
|
"händler in ihrer nähe", "probefahrt vereinbaren",
|
||||||
|
"anfrage an haendler", "anfrage an händler",
|
||||||
|
"konfigurator", "fahrzeug konfigurieren",
|
||||||
|
"ihre individuelle anfrage",
|
||||||
|
# OEM-Markennamen — sind Hersteller-Marken die ueblicherweise via
|
||||||
|
# Haendler vertreiben.
|
||||||
|
"bmw vertriebs", "audi vertriebs", "mercedes-benz vertriebs",
|
||||||
|
"volkswagen vertriebs", "porsche zentrum",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Indikatoren GEGEN no_direct_sales: echte Online-Shop-Funktionen.
|
||||||
|
_DIRECT_SALES_NEGATIVE = [
|
||||||
|
"in den warenkorb", "warenkorb hinzu", "zur kasse",
|
||||||
|
"jetzt kaufen", "kostenpflichtig bestellen",
|
||||||
|
"zahlungspflichtig bestellen", "sofort-kauf",
|
||||||
|
"online bestellen", "lieferadresse", "rechnungsadresse",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_no_direct_sales(full_text: str) -> bool:
|
||||||
|
"""Heuristik: erkennt OEM-Konfigurator-Sites die nicht direkt verkaufen."""
|
||||||
|
text = full_text.lower()
|
||||||
|
pos = sum(1 for k in _NO_DIRECT_SALES_POSITIVE if k in text)
|
||||||
|
neg = sum(1 for k in _DIRECT_SALES_NEGATIVE if k in text)
|
||||||
|
# Mindestens 3 Haendler-Indikatoren UND weniger Shop-Indikatoren als
|
||||||
|
# Haendler-Indikatoren. Vermeidet false-positive fuer Shops die
|
||||||
|
# zusaetzlich "Haendlersuche" als Filiale-Finder anbieten.
|
||||||
|
return pos >= 3 and pos > neg
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
"""
|
||||||
|
Zentraler User-Agent-Provider + Domain-Rate-Limiter fuer alle Crawls.
|
||||||
|
|
||||||
|
UA-Switch ist Trigger-gebunden an Firmengruendung:
|
||||||
|
- aktuell (Vor-Gruendung): generischer Headless-Chrome-UA
|
||||||
|
- nach Gruendung: env BREAKPILOT_BRANDED_UA=1 setzen
|
||||||
|
-> "BreakPilot-Compliance-Scanner/1.0 (+https://...)"
|
||||||
|
|
||||||
|
Memory: project_legal_contracts_2026_07.md (Punkt 0).
|
||||||
|
|
||||||
|
Rate-Limit:
|
||||||
|
- Default 1 req/sec/Domain, max 2 concurrent pro Domain.
|
||||||
|
- Saving-Scan-Funnel separat: max 1 vollstaendiger Run / Domain / 24h.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from collections import defaultdict
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
|
||||||
|
_BRANDED_UA = (
|
||||||
|
"BreakPilot-Compliance-Scanner/1.0 "
|
||||||
|
"(+https://breakpilot.ai/scanner)"
|
||||||
|
)
|
||||||
|
_NEUTRAL_UA = (
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def crawler_user_agent() -> str:
|
||||||
|
"""Aktueller UA-String fuer alle ausgehenden Crawls.
|
||||||
|
|
||||||
|
Switcht auf den Markennamen sobald BREAKPILOT_BRANDED_UA=1 gesetzt
|
||||||
|
wird (nach Firmengruendung — siehe Memory).
|
||||||
|
"""
|
||||||
|
branded = (os.getenv("BREAKPILOT_BRANDED_UA") or "").strip().lower()
|
||||||
|
if branded in ("1", "true", "yes"):
|
||||||
|
return _BRANDED_UA
|
||||||
|
return _NEUTRAL_UA
|
||||||
|
|
||||||
|
|
||||||
|
def default_request_headers() -> dict:
|
||||||
|
"""Vollstaendiger Header-Satz fuer httpx-Calls."""
|
||||||
|
return {
|
||||||
|
"User-Agent": crawler_user_agent(),
|
||||||
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def base_domain_of(url_or_host: str) -> str:
|
||||||
|
if not url_or_host:
|
||||||
|
return ""
|
||||||
|
if "://" not in url_or_host:
|
||||||
|
url_or_host = "https://" + url_or_host
|
||||||
|
netloc = urlparse(url_or_host).netloc.lower()
|
||||||
|
return netloc.replace("www.", "") or url_or_host
|
||||||
|
|
||||||
|
|
||||||
|
# --- per-Domain Rate-Limit ----------------------------------------------
|
||||||
|
|
||||||
|
_MIN_INTERVAL_S = 1.0 # 1 req/sec/Domain
|
||||||
|
_MAX_CONCURRENT_PER_DOMAIN = 2
|
||||||
|
|
||||||
|
_last_request_at: dict[str, float] = defaultdict(float)
|
||||||
|
_semaphores: dict[str, asyncio.Semaphore] = {}
|
||||||
|
_locks_lock = asyncio.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_semaphore(domain: str) -> asyncio.Semaphore:
|
||||||
|
async with _locks_lock:
|
||||||
|
sem = _semaphores.get(domain)
|
||||||
|
if sem is None:
|
||||||
|
sem = asyncio.Semaphore(_MAX_CONCURRENT_PER_DOMAIN)
|
||||||
|
_semaphores[domain] = sem
|
||||||
|
return sem
|
||||||
|
|
||||||
|
|
||||||
|
class DomainRateLimiter:
|
||||||
|
"""Async-Context: warte vor Request + halte concurrent-Slot.
|
||||||
|
|
||||||
|
async with DomainRateLimiter(url):
|
||||||
|
resp = await client.get(url)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, url_or_domain: str):
|
||||||
|
self.domain = base_domain_of(url_or_domain)
|
||||||
|
|
||||||
|
async def __aenter__(self):
|
||||||
|
sem = await _get_semaphore(self.domain)
|
||||||
|
await sem.acquire()
|
||||||
|
last = _last_request_at[self.domain]
|
||||||
|
wait = (last + _MIN_INTERVAL_S) - time.monotonic()
|
||||||
|
if wait > 0:
|
||||||
|
await asyncio.sleep(wait)
|
||||||
|
_last_request_at[self.domain] = time.monotonic()
|
||||||
|
self._sem = sem
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, exc_type, exc, tb):
|
||||||
|
self._sem.release()
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# --- per-Domain "1 full run / 24h" (Saving-Scan) -----------------------
|
||||||
|
|
||||||
|
_DB_PATH = os.getenv("COMPLIANCE_AUDIT_DB", "/data/compliance_audits.db")
|
||||||
|
_SAVING_SCAN_INTERVAL_S = 24 * 3600
|
||||||
|
|
||||||
|
|
||||||
|
def saving_scan_allowed(domain_or_url: str) -> tuple[bool, int]:
|
||||||
|
"""True wenn fuer diese Domain in den letzten 24h kein Saving-Scan lief.
|
||||||
|
|
||||||
|
Liest aus compliance_audit_log.check_runs (existierende Tabelle).
|
||||||
|
Liefert (allowed, seconds_until_allowed).
|
||||||
|
"""
|
||||||
|
import sqlite3
|
||||||
|
domain = base_domain_of(domain_or_url)
|
||||||
|
if not domain:
|
||||||
|
return True, 0
|
||||||
|
try:
|
||||||
|
with sqlite3.connect(_DB_PATH) as conn:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT MAX(ts) FROM check_runs WHERE base_domain=?",
|
||||||
|
(domain,),
|
||||||
|
).fetchone()
|
||||||
|
last = row[0] if row else None
|
||||||
|
if not last:
|
||||||
|
return True, 0
|
||||||
|
from datetime import datetime
|
||||||
|
elapsed = time.time() - datetime.fromisoformat(last).timestamp()
|
||||||
|
if elapsed >= _SAVING_SCAN_INTERVAL_S:
|
||||||
|
return True, 0
|
||||||
|
return False, int(_SAVING_SCAN_INTERVAL_S - elapsed)
|
||||||
|
except Exception:
|
||||||
|
return True, 0
|
||||||
@@ -129,20 +129,29 @@ def classify_cookie(cookie_name: str) -> tuple[str, str]:
|
|||||||
|
|
||||||
|
|
||||||
def annotate_vendor_cookies(vendor: dict) -> dict:
|
def annotate_vendor_cookies(vendor: dict) -> dict:
|
||||||
"""Enrich a vendor record with functional_role per cookie."""
|
"""Enrich a vendor record with functional_role + KB knowledge per cookie."""
|
||||||
|
from compliance.services.cookie_knowledge import (
|
||||||
|
lookup_cookie, summarize_compliance_risk,
|
||||||
|
)
|
||||||
cookies = vendor.get("cookies") or []
|
cookies = vendor.get("cookies") or []
|
||||||
annotated = []
|
annotated = []
|
||||||
role_counts: dict[str, int] = {}
|
role_counts: dict[str, int] = {}
|
||||||
for c in cookies:
|
for c in cookies:
|
||||||
role, impact = classify_cookie(c.get("name", ""))
|
role, impact = classify_cookie(c.get("name", ""))
|
||||||
annotated.append({**c, "functional_role": role, "blocking_impact": impact})
|
knowledge = lookup_cookie(c.get("name", ""))
|
||||||
|
entry = {**c, "functional_role": role, "blocking_impact": impact}
|
||||||
|
if knowledge:
|
||||||
|
entry["knowledge"] = knowledge
|
||||||
|
annotated.append(entry)
|
||||||
role_counts[role] = role_counts.get(role, 0) + 1
|
role_counts[role] = role_counts.get(role, 0) + 1
|
||||||
return {
|
out = {
|
||||||
**vendor,
|
**vendor,
|
||||||
"cookies": annotated,
|
"cookies": annotated,
|
||||||
"role_distribution": role_counts,
|
"role_distribution": role_counts,
|
||||||
"role_labels": {r: _FUNCTIONAL_LABEL.get(r, r) for r in role_counts},
|
"role_labels": {r: _FUNCTIONAL_LABEL.get(r, r) for r in role_counts},
|
||||||
}
|
}
|
||||||
|
out["compliance_risk"] = summarize_compliance_risk(out)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def aggregate_cookie_purposes(vendors: Iterable[dict]) -> dict:
|
def aggregate_cookie_purposes(vendors: Iterable[dict]) -> dict:
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""
|
||||||
|
Cookie-Knowledge Facade — vereint die Basis-KB (cookie_knowledge_db) mit
|
||||||
|
der Erweiterung (cookie_knowledge_extended) hinter einer einzigen API.
|
||||||
|
|
||||||
|
Caller sollten von hier importieren statt von einer der beiden Sub-DBs.
|
||||||
|
|
||||||
|
from compliance.services.cookie_knowledge import (
|
||||||
|
lookup_cookie,
|
||||||
|
enrich_vendor_with_knowledge,
|
||||||
|
summarize_compliance_risk,
|
||||||
|
compliance_risk_label,
|
||||||
|
)
|
||||||
|
|
||||||
|
Lookup-Reihenfolge: Extended (kuratiert, juenger) vor Base. Dadurch
|
||||||
|
koennen wir Eintraege ueberschreiben ohne die Base zu touchen.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from compliance.services.cookie_knowledge_db import (
|
||||||
|
CookieKnowledge,
|
||||||
|
lookup_cookie as _lookup_base,
|
||||||
|
)
|
||||||
|
from compliance.services.cookie_knowledge_extended import (
|
||||||
|
KB_EXT,
|
||||||
|
lookup_cookie_extended,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def lookup_cookie(name: str) -> CookieKnowledge | None:
|
||||||
|
"""Resolve cookie name to enriched knowledge — extended overrides base."""
|
||||||
|
return lookup_cookie_extended(name) or _lookup_base(name)
|
||||||
|
|
||||||
|
|
||||||
|
def enrich_vendor_with_knowledge(vendor: dict) -> dict:
|
||||||
|
"""Add per-cookie knowledge dict + per-vendor risk summary."""
|
||||||
|
cookies = vendor.get("cookies") or []
|
||||||
|
enriched = []
|
||||||
|
for c in cookies:
|
||||||
|
info = lookup_cookie(c.get("name", ""))
|
||||||
|
enriched.append({**c, "knowledge": info} if info else c)
|
||||||
|
out = {**vendor, "cookies": enriched}
|
||||||
|
out["compliance_risk"] = summarize_compliance_risk(out)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def summarize_compliance_risk(vendor: dict) -> dict:
|
||||||
|
"""Aggregate Re-ID risk + Schrems-II exposure across all cookies."""
|
||||||
|
cookies = vendor.get("cookies") or []
|
||||||
|
risk_counts = {"high": 0, "medium": 0, "low": 0}
|
||||||
|
schrems_affected = 0
|
||||||
|
strictly_necessary = 0
|
||||||
|
classified = 0
|
||||||
|
for c in cookies:
|
||||||
|
k = c.get("knowledge") or lookup_cookie(c.get("name", ""))
|
||||||
|
if not k:
|
||||||
|
continue
|
||||||
|
classified += 1
|
||||||
|
risk = (k.get("reid_risk") or "low").lower()
|
||||||
|
risk_counts[risk] = risk_counts.get(risk, 0) + 1
|
||||||
|
if "us" in (k.get("vendor_country") or "").lower() or \
|
||||||
|
"schrems" in (k.get("schrems_ii_status") or "").lower():
|
||||||
|
schrems_affected += 1
|
||||||
|
if k.get("technical_necessity") == "full":
|
||||||
|
strictly_necessary += 1
|
||||||
|
return {
|
||||||
|
"reid_risk_distribution": risk_counts,
|
||||||
|
"high_risk_cookie_count": risk_counts["high"],
|
||||||
|
"schrems_ii_affected_cookies": schrems_affected,
|
||||||
|
"strictly_necessary_cookies": strictly_necessary,
|
||||||
|
"total_classified": classified,
|
||||||
|
"label": compliance_risk_label({
|
||||||
|
"high_risk_cookie_count": risk_counts["high"],
|
||||||
|
"schrems_ii_affected_cookies": schrems_affected,
|
||||||
|
"total_classified": classified,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def compliance_risk_label(summary: dict) -> str:
|
||||||
|
"""Compact risk badge: 'kritisch' | 'hoch' | 'mittel' | 'gering' | 'unklar'."""
|
||||||
|
if not summary or not summary.get("total_classified"):
|
||||||
|
return "unklar"
|
||||||
|
high = summary.get("high_risk_cookie_count", 0)
|
||||||
|
schrems = summary.get("schrems_ii_affected_cookies", 0)
|
||||||
|
total = summary.get("total_classified", 0) or 1
|
||||||
|
if high >= 3 and schrems >= 2:
|
||||||
|
return "kritisch"
|
||||||
|
if high >= 2 or (high >= 1 and schrems >= 1):
|
||||||
|
return "hoch"
|
||||||
|
if high >= 1 or schrems >= 1:
|
||||||
|
return "mittel"
|
||||||
|
return "gering"
|
||||||
|
|
||||||
|
|
||||||
|
def kb_size() -> dict:
|
||||||
|
"""Diagnostik fuer den Admin/Health-Endpoint."""
|
||||||
|
from compliance.services.cookie_knowledge_db import KB as _KB_BASE
|
||||||
|
base_keys = set(_KB_BASE.keys())
|
||||||
|
ext_keys = set(KB_EXT.keys())
|
||||||
|
return {
|
||||||
|
"base_entries": len(base_keys),
|
||||||
|
"extended_entries": len(ext_keys),
|
||||||
|
"extended_overrides_base": len(base_keys & ext_keys),
|
||||||
|
"total_unique": len(base_keys | ext_keys),
|
||||||
|
}
|
||||||
@@ -0,0 +1,497 @@
|
|||||||
|
"""
|
||||||
|
Cookie-Knowledge Erweiterung — Adobe, Meta erweitert, Microsoft, LinkedIn,
|
||||||
|
TikTok, Salesforce/HubSpot/Marketo, Hotjar/Mouseflow/FullStory, Live-Chat,
|
||||||
|
Cloudflare/Akamai, Payment, CMP-eigene Cookies, EU-Analytics.
|
||||||
|
|
||||||
|
Hinweis zu Rechten: Eintraege enthalten ausschliesslich Identitaetsfelder
|
||||||
|
(Cookie-Name, Anbieter, Sitzland) + EIGENE Knappformulierungen + Verweise
|
||||||
|
auf oeffentliche EuGH-/CNIL-/EDPB-Quellen. KEINE 1:1-Kopien aus OneTrust,
|
||||||
|
Cookiepedia oder Vendor-eigenen Beschreibungstexten.
|
||||||
|
|
||||||
|
Quellen-Pointer: IAB TCF v2.2 Vendor List, CNIL Cookies & Trackers
|
||||||
|
Guidelines 2024, EDPB Guidelines 2/2023, EuGH-Rechtsprechung (Schrems II,
|
||||||
|
Planet49), DSK-Orientierungshilfen 2021/2024.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from compliance.services.cookie_knowledge_db import CookieKnowledge
|
||||||
|
|
||||||
|
|
||||||
|
_ADOBE_BASE = {
|
||||||
|
"vendor": "Adobe Inc.", "vendor_country": "US",
|
||||||
|
"schrems_ii_status": "Drittlandtransfer US. Mit DPF (2023) wieder "
|
||||||
|
"zulaessig; EU-Datenresidenz-Option in Adobe "
|
||||||
|
"Experience Platform verfuegbar.",
|
||||||
|
"eugh_rulings": [
|
||||||
|
"EuGH C-311/18 (Schrems II)",
|
||||||
|
"EDPB Recommendations 01/2020 — Supplementary Measures",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
_META_BASE = {
|
||||||
|
"vendor": "Meta Platforms Ireland Ltd.", "vendor_country": "IE",
|
||||||
|
"schrems_ii_status": "Verarbeitung in IE + US-Transfer. DPC Ireland "
|
||||||
|
"Bussgeld 2023 (€1,2 Mrd) wegen unzureichender "
|
||||||
|
"Schutzmassnahmen — DPF deckt seit 2023.",
|
||||||
|
"eugh_rulings": [
|
||||||
|
"EuGH C-311/18 (Schrems II)",
|
||||||
|
"DPC Ireland 2023 — Meta 1,2 Mrd. EUR",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
_MICROSOFT_BASE = {
|
||||||
|
"vendor": "Microsoft Corp.", "vendor_country": "US",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert; EU Data Boundary fuer Azure/365 "
|
||||||
|
"seit 2024 verfuegbar.",
|
||||||
|
"eugh_rulings": ["EuGH C-311/18 (Schrems II)"],
|
||||||
|
}
|
||||||
|
|
||||||
|
_LINKEDIN_BASE = {
|
||||||
|
"vendor": "LinkedIn Ireland Unlimited Co.", "vendor_country": "IE",
|
||||||
|
"schrems_ii_status": "Microsoft-Konzern, EU-Hauptsitz IE, Transfer US.",
|
||||||
|
"eugh_rulings": ["EuGH C-311/18 (Schrems II)"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
KB_EXT: dict[str, CookieKnowledge] = {
|
||||||
|
|
||||||
|
# --- Adobe Experience Cloud --------------------------------------
|
||||||
|
# AMCV_, s_cc, s_sq leben in Base-KB.
|
||||||
|
"demdex": {
|
||||||
|
**_ADOBE_BASE,
|
||||||
|
"vendor": "Adobe Inc. (Audience Manager)",
|
||||||
|
"exact_purpose": "Adobe Audience Manager DMP — Cross-Site-Profil "
|
||||||
|
"fuer Zielgruppen-Segmentierung.",
|
||||||
|
"data_collected": ["dpuuid", "segments"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"tcf_purpose_ids": [4, 9, 10],
|
||||||
|
"typical_lifetime": "180 Tage",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- Meta erweitert -----------------------------------------------
|
||||||
|
# fr, _fbc leben in Base-KB.
|
||||||
|
"datr": {
|
||||||
|
**_META_BASE,
|
||||||
|
"exact_purpose": "Facebook Browser-Identifier — Anti-Abuse/Bot-Schutz.",
|
||||||
|
"data_collected": ["browser_fingerprint_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"typical_lifetime": "2 Jahre",
|
||||||
|
"reid_risk": "high", "technical_necessity": "partial",
|
||||||
|
"notes": "Wird auch ohne Consent gesetzt; Meta argumentiert "
|
||||||
|
"Sicherheit. Trotzdem von DSK 2024 kritisch bewertet.",
|
||||||
|
},
|
||||||
|
# --- Microsoft / Bing ---------------------------------------------
|
||||||
|
# MUID lebt in Base-KB.
|
||||||
|
"MSCC": {
|
||||||
|
**_MICROSOFT_BASE,
|
||||||
|
"exact_purpose": "Microsoft Site Consent — Consent-Status-Speicherung "
|
||||||
|
"fuer Microsoft-eigene Properties.",
|
||||||
|
"data_collected": ["consent_string"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
"notes": "Strictly necessary nach §25(2) TDDDG.",
|
||||||
|
},
|
||||||
|
"ai_session": {
|
||||||
|
**_MICROSOFT_BASE,
|
||||||
|
"vendor": "Microsoft Corp. (Application Insights)",
|
||||||
|
"exact_purpose": "Azure Application Insights — Session-Tracking fuer "
|
||||||
|
"Telemetry.",
|
||||||
|
"data_collected": ["session_id"],
|
||||||
|
"typical_lifetime": "30 Minuten",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "partial",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- LinkedIn ------------------------------------------------------
|
||||||
|
"li_at": {
|
||||||
|
**_LINKEDIN_BASE,
|
||||||
|
"exact_purpose": "LinkedIn-Authentifizierung — Login-Session.",
|
||||||
|
"data_collected": ["auth_token"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "full",
|
||||||
|
"notes": "Nur fuer eingeloggte Nutzer; auf externer Site = "
|
||||||
|
"Insight Tag (siehe li_sugr).",
|
||||||
|
},
|
||||||
|
"li_sugr": {
|
||||||
|
**_LINKEDIN_BASE,
|
||||||
|
"exact_purpose": "LinkedIn Insight Tag — Browser-ID fuer "
|
||||||
|
"Conversion-Tracking + Werbe-Targeting.",
|
||||||
|
"data_collected": ["browser_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"tcf_purpose_ids": [7, 9, 10],
|
||||||
|
"typical_lifetime": "90 Tage",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
},
|
||||||
|
# bcookie, lidc leben in Base-KB.
|
||||||
|
|
||||||
|
# --- TikTok --------------------------------------------------------
|
||||||
|
"_ttp": {
|
||||||
|
"vendor": "TikTok Pte. Ltd.", "vendor_country": "SG/CN",
|
||||||
|
"exact_purpose": "TikTok Pixel — User-ID fuer Conversion-Tracking + "
|
||||||
|
"Werbeoptimierung.",
|
||||||
|
"data_collected": ["pixel_id", "browser_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"tcf_purpose_ids": [7, 9, 10],
|
||||||
|
"typical_lifetime": "13 Monate",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "Drittlandtransfer in Drittstaaten ohne "
|
||||||
|
"Angemessenheitsbeschluss. CNIL 2023 — "
|
||||||
|
"TikTok 5 Mio EUR Bussgeld.",
|
||||||
|
"eugh_rulings": [
|
||||||
|
"CNIL SAN-2022-027 — TikTok 5 Mio EUR",
|
||||||
|
"Italienische DPA 2024 — TikTok 10 Mio EUR",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"ttwid": {
|
||||||
|
"vendor": "TikTok Pte. Ltd.", "vendor_country": "SG/CN",
|
||||||
|
"exact_purpose": "TikTok Web-Identifier — eindeutige Browser-ID auch "
|
||||||
|
"ohne Login.",
|
||||||
|
"data_collected": ["ttwid"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "Wie _ttp.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- HubSpot / Marketo / Salesforce ------------------------------
|
||||||
|
"hubspotutk": {
|
||||||
|
"vendor": "HubSpot Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "HubSpot User-Token — Cross-Visit-Identitaet fuer "
|
||||||
|
"Lead-Tracking.",
|
||||||
|
"data_collected": ["user_token"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"tcf_purpose_ids": [7, 8],
|
||||||
|
"typical_lifetime": "6 Monate",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert.",
|
||||||
|
},
|
||||||
|
"__hssc": {
|
||||||
|
"vendor": "HubSpot Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "HubSpot Session-Tracking — Pageviews innerhalb "
|
||||||
|
"einer Session.",
|
||||||
|
"data_collected": ["session_count"],
|
||||||
|
"typical_lifetime": "30 Minuten",
|
||||||
|
"reid_risk": "low", "technical_necessity": "none",
|
||||||
|
},
|
||||||
|
"_mkto_trk": {
|
||||||
|
"vendor": "Adobe Inc. (Marketo)", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Marketo Munchkin-Tracker — Lead-Identifikation "
|
||||||
|
"fuer Marketing-Automation.",
|
||||||
|
"data_collected": ["munchkin_id", "session_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"typical_lifetime": "2 Jahre",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": _ADOBE_BASE["schrems_ii_status"],
|
||||||
|
},
|
||||||
|
"BrowserId_sec": {
|
||||||
|
"vendor": "Salesforce.com Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Salesforce Marketing Cloud Browser-Token — "
|
||||||
|
"Cross-Visit-Identifikation.",
|
||||||
|
"data_collected": ["browser_id"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- Session-Recording / Heatmaps ---------------------------------
|
||||||
|
"_hjSessionUser_": {
|
||||||
|
"vendor": "Hotjar Ltd.", "vendor_country": "MT",
|
||||||
|
"exact_purpose": "Hotjar User-ID — Cross-Visit-Identifikation fuer "
|
||||||
|
"Session-Recording + Heatmaps.",
|
||||||
|
"data_collected": ["user_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "EU (Malta) — kein Drittland. Aber: parent "
|
||||||
|
"Contentsquare (FR) hostet teilweise in US.",
|
||||||
|
"notes": "Suffix `<site_id>`. Pattern-Match noetig. "
|
||||||
|
"DSGVO-Aufzeichnung = Einwilligung pflichtig.",
|
||||||
|
"eu_alternative_vendor": "Mouseflow / Smartlook (CZ)",
|
||||||
|
},
|
||||||
|
"_hjSession_": {
|
||||||
|
"vendor": "Hotjar Ltd.", "vendor_country": "MT",
|
||||||
|
"exact_purpose": "Hotjar Session-Token — eindeutige Session-ID "
|
||||||
|
"innerhalb 30min Inaktivitaet.",
|
||||||
|
"data_collected": ["session_id"],
|
||||||
|
"typical_lifetime": "30 Minuten",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "none",
|
||||||
|
},
|
||||||
|
"fs_uid": {
|
||||||
|
"vendor": "FullStory Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "FullStory User-ID — Cross-Visit-Identifikation "
|
||||||
|
"fuer Session-Replay.",
|
||||||
|
"data_collected": ["user_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert. EU-Region verfuegbar (opt-in).",
|
||||||
|
},
|
||||||
|
"mf_user": {
|
||||||
|
"vendor": "Mouseflow Aps", "vendor_country": "DK",
|
||||||
|
"exact_purpose": "Mouseflow User-ID — Cross-Visit-Identifikation fuer "
|
||||||
|
"Heatmap + Recording.",
|
||||||
|
"data_collected": ["user_id"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "EU (DK) — kein Drittland.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- Live-Chat ----------------------------------------------------
|
||||||
|
"intercom-id-": {
|
||||||
|
"vendor": "Intercom Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Intercom Visitor-ID — Wiedererkennung anonymer "
|
||||||
|
"Besucher fuer Chat-History.",
|
||||||
|
"data_collected": ["visitor_id"],
|
||||||
|
"typical_lifetime": "9 Monate",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "partial",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert; EU-Datenresidenz optional.",
|
||||||
|
"notes": "Suffix `<app_id>`. Pattern-Match noetig.",
|
||||||
|
},
|
||||||
|
"driftt_aid": {
|
||||||
|
"vendor": "Salesforce.com Inc. (Drift)", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Drift Anonymous-Visitor-ID fuer Chat-Personalisierung.",
|
||||||
|
"data_collected": ["visitor_id"],
|
||||||
|
"typical_lifetime": "2 Jahre",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "partial",
|
||||||
|
},
|
||||||
|
"__zlcmid": {
|
||||||
|
"vendor": "Zendesk Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Zendesk Chat Visitor-ID fuer Session-Tracking.",
|
||||||
|
"data_collected": ["chat_visitor_id"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "partial",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert; EU-Datacenter optional.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- CDN / Sicherheit (strictly necessary) -----------------------
|
||||||
|
# __cf_bm, cf_clearance leben in Base-KB.
|
||||||
|
"AKA_A2": {
|
||||||
|
"vendor": "Akamai Technologies Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Akamai Adaptive Acceleration — geroutete Best-Path-"
|
||||||
|
"Optimierung.",
|
||||||
|
"data_collected": ["a2_route"],
|
||||||
|
"typical_lifetime": "1 Stunde",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- Payment (strictly necessary fuer Checkout) ------------------
|
||||||
|
"__stripe_mid": {
|
||||||
|
"vendor": "Stripe Payments Europe Ltd.", "vendor_country": "IE",
|
||||||
|
"exact_purpose": "Stripe Fraud-Detection Merchant-ID — Risiko-Scoring "
|
||||||
|
"fuer Zahlungs-Authentifizierung.",
|
||||||
|
"data_collected": ["merchant_visitor_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
"schrems_ii_status": "EU (IE) — kein Drittland.",
|
||||||
|
"notes": "Strictly necessary nach §25(2) TDDDG fuer Zahlungsabwicklung.",
|
||||||
|
},
|
||||||
|
"__stripe_sid": {
|
||||||
|
"vendor": "Stripe Payments Europe Ltd.", "vendor_country": "IE",
|
||||||
|
"exact_purpose": "Stripe Session-ID — temporaere Zahlungs-Session.",
|
||||||
|
"data_collected": ["session_id"],
|
||||||
|
"typical_lifetime": "30 Minuten",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- CMP-eigene Cookies (strictly necessary) ---------------------
|
||||||
|
"CookieConsent": {
|
||||||
|
"vendor": "Cybot A/S (Cookiebot)", "vendor_country": "DK",
|
||||||
|
"exact_purpose": "Cookiebot Consent-Speicherung — gewaehlte "
|
||||||
|
"Kategorien + Zeitstempel.",
|
||||||
|
"data_collected": ["consent_categories", "consent_timestamp"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
"schrems_ii_status": "EU (DK). Wenn EU-Cloud, kein Drittland.",
|
||||||
|
},
|
||||||
|
"OptanonConsent": {
|
||||||
|
"vendor": "OneTrust LLC", "vendor_country": "US",
|
||||||
|
"exact_purpose": "OneTrust Consent-Speicherung — Kategorien + "
|
||||||
|
"Vendor-Liste + Zeitstempel.",
|
||||||
|
"data_collected": ["consent_categories", "consent_string"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert; EU-Cloud optional.",
|
||||||
|
},
|
||||||
|
"OptanonAlertBoxClosed": {
|
||||||
|
"vendor": "OneTrust LLC", "vendor_country": "US",
|
||||||
|
"exact_purpose": "OneTrust UI-Flag — verhindert Re-Display des "
|
||||||
|
"Banners nach Schliessung.",
|
||||||
|
"data_collected": ["closed_timestamp"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
},
|
||||||
|
"usercentrics-uuid": {
|
||||||
|
"vendor": "Usercentrics GmbH", "vendor_country": "DE",
|
||||||
|
"exact_purpose": "Usercentrics Consent-Speicherung — UUID-basiert.",
|
||||||
|
"data_collected": ["consent_uuid", "consent_settings"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "low", "technical_necessity": "full",
|
||||||
|
"schrems_ii_status": "DE — kein Drittland.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- Weitere Social / Werbeplattformen ---------------------------
|
||||||
|
# _pin_unauth lebt in Base-KB.
|
||||||
|
"_scid": {
|
||||||
|
"vendor": "Snap Group Ltd.", "vendor_country": "GB/US",
|
||||||
|
"exact_purpose": "Snapchat Pixel — Conversion-Tracking fuer "
|
||||||
|
"Snap Ads.",
|
||||||
|
"data_collected": ["snap_visitor_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"tcf_purpose_ids": [7, 9, 10],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "Drittlandtransfer; UK seit 2021 mit "
|
||||||
|
"Angemessenheitsbeschluss.",
|
||||||
|
},
|
||||||
|
"guest_id": {
|
||||||
|
"vendor": "X Corp. (Twitter)", "vendor_country": "US",
|
||||||
|
"exact_purpose": "X/Twitter Guest-Identifier — Tracking nicht "
|
||||||
|
"eingeloggter Besucher inkl. Embeds.",
|
||||||
|
"data_collected": ["guest_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"tcf_purpose_ids": [4, 9, 10],
|
||||||
|
"typical_lifetime": "2 Jahre",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-Status unklar seit Eigentuemerwechsel 2022. "
|
||||||
|
"Erhoehtes Risiko, EDPB beobachtet.",
|
||||||
|
},
|
||||||
|
"VISITOR_INFO1_LIVE": {
|
||||||
|
"vendor": "Google Ireland Ltd. (YouTube)", "vendor_country": "IE",
|
||||||
|
"exact_purpose": "YouTube Embed Visitor-ID — Bandbreiten-Optimierung "
|
||||||
|
"+ Empfehlungsalgorithmus.",
|
||||||
|
"data_collected": ["youtube_visitor_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"tcf_purpose_ids": [8, 10],
|
||||||
|
"typical_lifetime": "6 Monate",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"notes": "YouTube-NoCookie-Domain (youtube-nocookie.com) reduziert "
|
||||||
|
"Tracking — DSGVO-konformer.",
|
||||||
|
},
|
||||||
|
"vuid": {
|
||||||
|
"vendor": "Vimeo Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Vimeo User-Identifier — Wiedererkennung "
|
||||||
|
"wiederkehrender Besucher fuer Statistik.",
|
||||||
|
"data_collected": ["vimeo_user_id"],
|
||||||
|
"typical_lifetime": "2 Jahre",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- Marketing-Automation / Email --------------------------------
|
||||||
|
"__kla_id": {
|
||||||
|
"vendor": "Klaviyo Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Klaviyo Visitor-Tracking — fuer E-Mail-Marketing-"
|
||||||
|
"Attribution.",
|
||||||
|
"data_collected": ["klaviyo_id"],
|
||||||
|
"ip_relevant": True,
|
||||||
|
"typical_lifetime": "2 Jahre",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert.",
|
||||||
|
},
|
||||||
|
"_mcid": {
|
||||||
|
"vendor": "Intuit Mailchimp", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Mailchimp Email-Click-Tracking — Verknuepft "
|
||||||
|
"Pageviews mit gesendeter Kampagne.",
|
||||||
|
"data_collected": ["mc_email_id"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- Product-Analytics / CDP -------------------------------------
|
||||||
|
"mp_": {
|
||||||
|
"vendor": "Mixpanel Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Mixpanel Distinct-ID + Properties — "
|
||||||
|
"Pseudonyme Event-Analytics.",
|
||||||
|
"data_collected": ["distinct_id", "properties"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert; EU-Residency optional.",
|
||||||
|
"notes": "Suffix `<token>_mixpanel`. Pattern-Match noetig.",
|
||||||
|
},
|
||||||
|
"ajs_anonymous_id": {
|
||||||
|
"vendor": "Twilio Inc. (Segment)", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Segment Anonymous-ID — Cross-Device-Identitaet "
|
||||||
|
"vor Login.",
|
||||||
|
"data_collected": ["anonymous_id"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert; EU-Datenresidenz optional.",
|
||||||
|
},
|
||||||
|
"AMP_": {
|
||||||
|
"vendor": "Amplitude Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Amplitude Device-ID — Cross-Session-Identitaet "
|
||||||
|
"fuer Product-Analytics.",
|
||||||
|
"data_collected": ["device_id", "session_id"],
|
||||||
|
"typical_lifetime": "1 Jahr",
|
||||||
|
"reid_risk": "high", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert.",
|
||||||
|
"notes": "Suffix `<api_key>`. Pattern-Match noetig.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- A/B-Testing -------------------------------------------------
|
||||||
|
"optimizelyEndUserId": {
|
||||||
|
"vendor": "Optimizely Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Optimizely End-User-ID — konsistente "
|
||||||
|
"Experiment-Zuteilung pro Besucher.",
|
||||||
|
"data_collected": ["end_user_id", "variation_assignments"],
|
||||||
|
"typical_lifetime": "6 Monate",
|
||||||
|
"reid_risk": "medium", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "DPF-zertifiziert.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- RUM / Monitoring (oft strictly necessary diskutiert) --------
|
||||||
|
"_dd_s": {
|
||||||
|
"vendor": "Datadog Inc.", "vendor_country": "US",
|
||||||
|
"exact_purpose": "Datadog RUM Session-Tracking — Performance- "
|
||||||
|
"Monitoring + Fehler-Telemetrie.",
|
||||||
|
"data_collected": ["session_id", "session_type"],
|
||||||
|
"typical_lifetime": "15 Minuten",
|
||||||
|
"reid_risk": "low", "technical_necessity": "partial",
|
||||||
|
"schrems_ii_status": "EU-Region (Frankfurt) verfuegbar.",
|
||||||
|
"notes": "Bei reiner Server-/Fehler-Telemetrie ohne Cross-Site-"
|
||||||
|
"Tracking Argument fuer berechtigtes Interesse moeglich.",
|
||||||
|
},
|
||||||
|
|
||||||
|
# --- EU-Analytics-Alternativen -----------------------------------
|
||||||
|
"_pk_ref": {
|
||||||
|
"vendor": "InnoCraft Ltd. (Matomo)", "vendor_country": "NZ",
|
||||||
|
"exact_purpose": "Matomo Referrer-Tracking — Quelle des Besuchs.",
|
||||||
|
"data_collected": ["referrer", "campaign"],
|
||||||
|
"typical_lifetime": "6 Monate",
|
||||||
|
"reid_risk": "low", "technical_necessity": "none",
|
||||||
|
"schrems_ii_status": "NZ hat Angemessenheitsbeschluss (2012). "
|
||||||
|
"Bei On-Premise-Hosting kein Transfer.",
|
||||||
|
"notes": "Self-Hosting empfohlen — dann zeroes Drittland.",
|
||||||
|
},
|
||||||
|
"_pk_cvar": {
|
||||||
|
"vendor": "InnoCraft Ltd. (Matomo)", "vendor_country": "NZ",
|
||||||
|
"exact_purpose": "Matomo Custom-Variables — pro Visit konfigurierbar.",
|
||||||
|
"data_collected": ["custom_vars"],
|
||||||
|
"typical_lifetime": "30 Minuten",
|
||||||
|
"reid_risk": "low", "technical_necessity": "none",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Pattern-Lookups fuer dynamische Cookie-Namen
|
||||||
|
_EXT_PATTERNS: list[tuple[str, str]] = [
|
||||||
|
(r"^_hjSessionUser_", "_hjSessionUser_"),
|
||||||
|
(r"^_hjSession_", "_hjSession_"),
|
||||||
|
(r"^intercom-id-", "intercom-id-"),
|
||||||
|
(r"^mp_", "mp_"),
|
||||||
|
(r"^AMP_", "AMP_"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def lookup_cookie_extended(name: str) -> CookieKnowledge | None:
|
||||||
|
"""Lookup in der KB_EXT (Extension). None wenn nicht gefunden."""
|
||||||
|
import re
|
||||||
|
if not name: return None # noqa: E701
|
||||||
|
if name in KB_EXT: return KB_EXT[name] # noqa: E701
|
||||||
|
for pat, key in _EXT_PATTERNS:
|
||||||
|
if re.search(pat, name): return KB_EXT.get(key) # noqa: E701
|
||||||
|
base = name.split(".", 1)[0]
|
||||||
|
if base != name and base in KB_EXT: return KB_EXT[base] # noqa: E701
|
||||||
|
return None
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
"""
|
||||||
|
TDM-Reservation-Check (§ 44b UrhG / EU CDSM Art. 4).
|
||||||
|
|
||||||
|
Prueft pro Domain ob ein maschinenlesbarer Nutzungsvorbehalt fuer
|
||||||
|
Text-and-Data-Mining gesetzt ist. Quellen:
|
||||||
|
1. robots.txt — User-agent: * Disallow: / (oder spezifisch fuer uns)
|
||||||
|
2. /ai.txt — neuer OpenAI-Standard
|
||||||
|
3. HTTP-Header `tdm-reservation: 1` auf Homepage
|
||||||
|
4. HTML <meta name="tdm-reservation" content="1"> auf Homepage
|
||||||
|
5. HTML <meta name="robots" content="noai|noimageai"> Tags
|
||||||
|
|
||||||
|
Status-Interpretation:
|
||||||
|
status=allowed -> kein Vorbehalt, crawlbar
|
||||||
|
status=reserved -> expliziter Vorbehalt, NICHT crawlen
|
||||||
|
status=denied -> robots.txt-Zugriff aktiv blockiert (403/401)
|
||||||
|
=> konservativ: NICHT crawlen
|
||||||
|
status=unknown -> Server-Error (500/timeout/DNS) auf robots.txt
|
||||||
|
=> crawlbar, aber 24h-Recheck markiert
|
||||||
|
|
||||||
|
Cache via sidecar SQLite (gleiche DB wie compliance_audit_log), 24h TTL.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Literal
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DB_PATH = os.getenv("COMPLIANCE_AUDIT_DB", "/data/compliance_audits.db")
|
||||||
|
CACHE_TTL_SECONDS = 24 * 3600
|
||||||
|
|
||||||
|
Status = Literal["allowed", "reserved", "denied", "unknown"]
|
||||||
|
|
||||||
|
_DEFAULT_UA = (
|
||||||
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_cache_table() -> None:
|
||||||
|
Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.executescript("""
|
||||||
|
CREATE TABLE IF NOT EXISTS tdm_reservation_cache (
|
||||||
|
domain TEXT PRIMARY KEY,
|
||||||
|
ts TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
signals TEXT NOT NULL -- JSON list[dict]
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tdm_ts ON tdm_reservation_cache(ts);
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_get(domain: str) -> dict | None:
|
||||||
|
try:
|
||||||
|
_ensure_cache_table()
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT * FROM tdm_reservation_cache WHERE domain=?", (domain,),
|
||||||
|
).fetchone()
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
ts = datetime.fromisoformat(row["ts"]).timestamp()
|
||||||
|
if time.time() - ts > CACHE_TTL_SECONDS:
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
"domain": domain,
|
||||||
|
"status": row["status"],
|
||||||
|
"signals": json.loads(row["signals"]),
|
||||||
|
"cached": True,
|
||||||
|
"ts": row["ts"],
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("tdm cache_get failed for %s: %s", domain, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_put(domain: str, status: Status, signals: list[dict]) -> None:
|
||||||
|
try:
|
||||||
|
_ensure_cache_table()
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR REPLACE INTO tdm_reservation_cache "
|
||||||
|
"(domain, ts, status, signals) VALUES (?, ?, ?, ?)",
|
||||||
|
(
|
||||||
|
domain,
|
||||||
|
datetime.now(timezone.utc).isoformat(),
|
||||||
|
status,
|
||||||
|
json.dumps(signals, ensure_ascii=False),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("tdm cache_put failed for %s: %s", domain, e)
|
||||||
|
|
||||||
|
|
||||||
|
def _base_domain(url_or_domain: str) -> str:
|
||||||
|
if not url_or_domain:
|
||||||
|
return ""
|
||||||
|
if "://" not in url_or_domain:
|
||||||
|
url_or_domain = "https://" + url_or_domain
|
||||||
|
netloc = urlparse(url_or_domain).netloc.lower()
|
||||||
|
return netloc.replace("www.", "")
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch_status(client: httpx.AsyncClient, url: str) -> tuple[int, str, dict]:
|
||||||
|
"""Return (status_code, body, headers). Body capped at 16 KiB."""
|
||||||
|
try:
|
||||||
|
resp = await client.get(url)
|
||||||
|
body = resp.text[:16384] if resp.content else ""
|
||||||
|
return resp.status_code, body, dict(resp.headers)
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("tdm fetch %s failed: %s", url, e)
|
||||||
|
return 0, "", {}
|
||||||
|
|
||||||
|
|
||||||
|
def _robots_disallows_us(body: str) -> bool:
|
||||||
|
"""Parse robots.txt — true if our group has Disallow: /."""
|
||||||
|
if not body:
|
||||||
|
return False
|
||||||
|
relevant_groups = ["*", "claudebot", "anthropic-ai", "gptbot",
|
||||||
|
"google-extended", "ccbot", "breakpilot"]
|
||||||
|
current_uas: list[str] = []
|
||||||
|
in_our_group = False
|
||||||
|
for raw in body.splitlines():
|
||||||
|
line = raw.split("#", 1)[0].strip()
|
||||||
|
if not line:
|
||||||
|
in_our_group = False
|
||||||
|
current_uas = []
|
||||||
|
continue
|
||||||
|
if ":" not in line:
|
||||||
|
continue
|
||||||
|
key, val = (s.strip().lower() for s in line.split(":", 1))
|
||||||
|
if key == "user-agent":
|
||||||
|
current_uas.append(val)
|
||||||
|
in_our_group = any(ua in relevant_groups for ua in current_uas)
|
||||||
|
elif key == "disallow" and in_our_group:
|
||||||
|
if val == "/" or val == "":
|
||||||
|
if val == "/":
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _meta_has_reservation(body: str) -> bool:
|
||||||
|
"""Detect <meta name="tdm-reservation|robots|googlebot"> with noai/noimageai/1."""
|
||||||
|
low = body.lower()
|
||||||
|
needles = [
|
||||||
|
'name="tdm-reservation" content="1"',
|
||||||
|
"name='tdm-reservation' content='1'",
|
||||||
|
'"noai"', '"noimageai"',
|
||||||
|
"content=\"noai", "content='noai",
|
||||||
|
]
|
||||||
|
return any(n in low for n in needles)
|
||||||
|
|
||||||
|
|
||||||
|
async def check_tdm_reservation(domain_or_url: str) -> dict:
|
||||||
|
"""Probe a domain for machine-readable TDM reservations.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{
|
||||||
|
domain, status, signals: [{src, detail}], cached, ts
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
domain = _base_domain(domain_or_url)
|
||||||
|
if not domain:
|
||||||
|
return {"domain": "", "status": "unknown", "signals": [], "cached": False}
|
||||||
|
|
||||||
|
cached = _cache_get(domain)
|
||||||
|
if cached:
|
||||||
|
return cached
|
||||||
|
|
||||||
|
signals: list[dict] = []
|
||||||
|
status: Status = "allowed"
|
||||||
|
|
||||||
|
headers = {"User-Agent": _DEFAULT_UA, "Accept": "*/*"}
|
||||||
|
async with httpx.AsyncClient(
|
||||||
|
timeout=12.0, follow_redirects=True, headers=headers,
|
||||||
|
) as client:
|
||||||
|
for scheme in ("https", "http"):
|
||||||
|
r_code, r_body, _ = await _fetch_status(
|
||||||
|
client, f"{scheme}://www.{domain}/robots.txt",
|
||||||
|
)
|
||||||
|
if r_code == 0 and scheme == "https":
|
||||||
|
continue
|
||||||
|
signals.append({"src": "robots.txt", "status_code": r_code,
|
||||||
|
"scheme": scheme})
|
||||||
|
if r_code in (401, 403):
|
||||||
|
status = "denied"
|
||||||
|
elif r_code == 200 and _robots_disallows_us(r_body):
|
||||||
|
status = "reserved"
|
||||||
|
signals[-1]["detail"] = "Disallow: / for relevant UA group"
|
||||||
|
elif r_code not in (200, 404):
|
||||||
|
status = "unknown"
|
||||||
|
break
|
||||||
|
|
||||||
|
if status == "allowed":
|
||||||
|
ai_code, _, _ = await _fetch_status(
|
||||||
|
client, f"https://www.{domain}/ai.txt",
|
||||||
|
)
|
||||||
|
if ai_code == 200:
|
||||||
|
status = "reserved"
|
||||||
|
signals.append({"src": "ai.txt", "status_code": 200,
|
||||||
|
"detail": "ai.txt present"})
|
||||||
|
|
||||||
|
if status == "allowed":
|
||||||
|
h_code, h_body, h_hdrs = await _fetch_status(
|
||||||
|
client, f"https://www.{domain}/",
|
||||||
|
)
|
||||||
|
if h_code == 200:
|
||||||
|
if h_hdrs.get("tdm-reservation") == "1":
|
||||||
|
status = "reserved"
|
||||||
|
signals.append({"src": "http-header",
|
||||||
|
"detail": "tdm-reservation: 1"})
|
||||||
|
elif _meta_has_reservation(h_body):
|
||||||
|
status = "reserved"
|
||||||
|
signals.append({"src": "html-meta",
|
||||||
|
"detail": "noai/tdm-reservation meta"})
|
||||||
|
|
||||||
|
_cache_put(domain, status, signals)
|
||||||
|
return {
|
||||||
|
"domain": domain,
|
||||||
|
"status": status,
|
||||||
|
"signals": signals,
|
||||||
|
"cached": False,
|
||||||
|
"ts": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def is_crawl_allowed(result: dict) -> bool:
|
||||||
|
"""Strict: only 'allowed' and 'unknown' are crawlable."""
|
||||||
|
return (result.get("status") or "unknown") in ("allowed", "unknown")
|
||||||
@@ -0,0 +1,277 @@
|
|||||||
|
"""
|
||||||
|
Aggregator: Doc-Check-Results + cmp_vendors + redundancy_report
|
||||||
|
-> einheitliche Finding-Records fuer unified_findings_store.
|
||||||
|
|
||||||
|
Speichert nur ABGELEITETE/normalisierte Findings (siehe Memory
|
||||||
|
'feedback_oem_data_legal.md'): keine rohen CMP-Cookie-Texte, keine
|
||||||
|
1:1-Spiegelung fremder Vendor-Listen — nur eigene Risk-/Status-Bewertung.
|
||||||
|
|
||||||
|
Hook:
|
||||||
|
from compliance.services.unified_findings_collector import collect
|
||||||
|
from compliance.services.unified_findings_store import record_findings
|
||||||
|
findings = collect(check_id, results, cmp_vendors, redundancy_report, doc_texts)
|
||||||
|
record_findings(check_id, findings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
_SEVERITY_DEFAULT = {
|
||||||
|
"mc": "MEDIUM",
|
||||||
|
"pflichtangabe": "MEDIUM",
|
||||||
|
"vendor": "MEDIUM",
|
||||||
|
"redundanz": "LOW",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Mapping cmp_vendor.flag → action_recipe key + Default-Severity
|
||||||
|
_VENDOR_FLAG_SEVERITY = {
|
||||||
|
"no_cookies_listed": ("HIGH", "Cookie-Auflistung fehlt"),
|
||||||
|
"no_country": ("MEDIUM", "Sitzland des Anbieters fehlt"),
|
||||||
|
"no_privacy_url": ("HIGH", "Datenschutzerklaerung des Anbieters fehlt"),
|
||||||
|
"broken_privacy_url": ("HIGH", "Datenschutz-URL nicht erreichbar"),
|
||||||
|
"no_opt_out_url": ("MEDIUM", "Widerspruchs-/Opt-Out-Link fehlt"),
|
||||||
|
"broken_opt_out": ("MEDIUM", "Opt-Out-Link nicht erreichbar"),
|
||||||
|
"no_name": ("HIGH", "Anbieter-Name fehlt"),
|
||||||
|
"no_purpose": ("HIGH", "Verarbeitungszweck fehlt"),
|
||||||
|
"cookies_no_expiry": ("LOW", "Cookie-Speicherdauer fehlt"),
|
||||||
|
"cookies_no_names": ("LOW", "Cookie-Namen fehlen"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_recipe(key: str) -> dict:
|
||||||
|
"""Lookup mit lazy-import — recipes-Modul ist optional."""
|
||||||
|
try:
|
||||||
|
from compliance.services.finding_action_recipes import recipe_for
|
||||||
|
r = recipe_for(key)
|
||||||
|
return dict(r) if r else {}
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_anchor(label: str, doc_text: str, doc_id: str) -> dict:
|
||||||
|
"""Anchor-Lookup mit lazy-import + best-effort."""
|
||||||
|
if not label or not doc_text:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
from compliance.services.doc_anchor_locator import locate_anchor
|
||||||
|
a = locate_anchor(label, doc_text, doc_id)
|
||||||
|
return a or {}
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _from_doc_check(
|
||||||
|
check_id: str,
|
||||||
|
r: Any,
|
||||||
|
doc_text: str,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Convert one DocCheckResult into unified-finding rows."""
|
||||||
|
out: list[dict] = []
|
||||||
|
if r.error and r.error.startswith("Nicht anwendbar"):
|
||||||
|
out.append({
|
||||||
|
"source_type": "pflichtangabe",
|
||||||
|
"doc_type": r.doc_type,
|
||||||
|
"severity": "INFO",
|
||||||
|
"status": "na",
|
||||||
|
"regulation": "",
|
||||||
|
"label": f"{r.label}: {r.error}",
|
||||||
|
"hint": r.error,
|
||||||
|
"action_recipe": {},
|
||||||
|
"payload": {"scenario": r.scenario},
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
if r.error:
|
||||||
|
out.append({
|
||||||
|
"source_type": "pflichtangabe",
|
||||||
|
"doc_type": r.doc_type,
|
||||||
|
"severity": "HIGH",
|
||||||
|
"status": "failed",
|
||||||
|
"regulation": "",
|
||||||
|
"label": f"{r.label}: Dokument nicht erreichbar",
|
||||||
|
"hint": r.error[:400],
|
||||||
|
"action_recipe": {},
|
||||||
|
"payload": {},
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
for c in (r.checks or []):
|
||||||
|
is_mc = (c.id or "").startswith("mc-")
|
||||||
|
source = "mc" if is_mc else "pflichtangabe"
|
||||||
|
if c.passed:
|
||||||
|
status = "passed"
|
||||||
|
elif c.skipped:
|
||||||
|
status = "skipped"
|
||||||
|
else:
|
||||||
|
status = "failed"
|
||||||
|
severity = (c.severity or _SEVERITY_DEFAULT[source]).upper()
|
||||||
|
# Nur fuer Fails Anchor + Recipe — Pass-Eintraege halten wir mager
|
||||||
|
recipe: dict = {}
|
||||||
|
anchor: dict = {}
|
||||||
|
if status == "failed":
|
||||||
|
# Recipe per Label-Substring (mehr als nur exakte Keys)
|
||||||
|
recipe = _safe_recipe(c.label or "") or _safe_recipe(c.id or "")
|
||||||
|
anchor = _safe_anchor(c.label or "", doc_text, r.doc_type)
|
||||||
|
out.append({
|
||||||
|
"source_type": source,
|
||||||
|
"doc_type": r.doc_type,
|
||||||
|
"severity": severity,
|
||||||
|
"status": status,
|
||||||
|
"regulation": c.regulation or "",
|
||||||
|
"label": c.label or "",
|
||||||
|
"hint": c.hint or "",
|
||||||
|
"action_recipe": recipe,
|
||||||
|
"anchor_excerpt": (anchor.get("anchor_phrase") or "")[:800],
|
||||||
|
"anchor_conf": _conf_to_score(anchor),
|
||||||
|
"payload": {
|
||||||
|
"mc_id": c.id,
|
||||||
|
"level": c.level,
|
||||||
|
"parent": c.parent,
|
||||||
|
"matched_text": (c.matched_text or "")[:300],
|
||||||
|
"article": c.article or "",
|
||||||
|
"anchor_method": anchor.get("method"),
|
||||||
|
"anchor_position": anchor.get("position_hint"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _conf_to_score(anchor: dict) -> float:
|
||||||
|
if not anchor:
|
||||||
|
return 0.0
|
||||||
|
try:
|
||||||
|
return float(anchor.get("score") or 0.0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def _from_vendors(check_id: str, vendors: list[dict]) -> list[dict]:
|
||||||
|
"""Per-vendor flag -> finding row."""
|
||||||
|
out: list[dict] = []
|
||||||
|
for v in vendors or []:
|
||||||
|
name = v.get("name") or v.get("vendor_name") or "Unbekannter Anbieter"
|
||||||
|
country = v.get("country") or ""
|
||||||
|
risk = v.get("compliance_risk") or {}
|
||||||
|
for flag in (v.get("compliance_flags") or v.get("flags") or []):
|
||||||
|
sev, label = _VENDOR_FLAG_SEVERITY.get(
|
||||||
|
flag, ("LOW", flag.replace("_", " ").title()),
|
||||||
|
)
|
||||||
|
out.append({
|
||||||
|
"source_type": "vendor",
|
||||||
|
"doc_type": "-",
|
||||||
|
"severity": sev,
|
||||||
|
"status": "failed",
|
||||||
|
"regulation": "DSGVO",
|
||||||
|
"label": f"{name} — {label}",
|
||||||
|
"hint": _vendor_hint(flag, name),
|
||||||
|
"action_recipe": _safe_recipe(flag),
|
||||||
|
"vendor_name": name,
|
||||||
|
"category": (v.get("category") or "")[:64],
|
||||||
|
"payload": {
|
||||||
|
"flag": flag,
|
||||||
|
"country": country,
|
||||||
|
"compliance_score": v.get("compliance_score"),
|
||||||
|
"category": v.get("category"),
|
||||||
|
"risk_label": risk.get("label"),
|
||||||
|
"high_risk_cookies": risk.get("high_risk_cookie_count"),
|
||||||
|
"schrems_ii_cookies": risk.get("schrems_ii_affected_cookies"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _vendor_hint(flag: str, name: str) -> str:
|
||||||
|
hints = {
|
||||||
|
"no_cookies_listed":
|
||||||
|
f"Bei '{name}' sind keine Cookies dokumentiert — DSK-Orientierungshilfe "
|
||||||
|
"verlangt Name + Zweck + Speicherdauer pro Cookie.",
|
||||||
|
"no_country":
|
||||||
|
f"Sitzland von '{name}' fehlt — bei Drittland-Anbieter "
|
||||||
|
"Art. 44 ff. DSGVO erforderlich.",
|
||||||
|
"no_privacy_url":
|
||||||
|
f"Link zur Datenschutzerklaerung von '{name}' fehlt — Art. 13 Abs. 1 lit. e.",
|
||||||
|
"broken_privacy_url":
|
||||||
|
f"Privacy-URL von '{name}' nicht erreichbar (404/Timeout).",
|
||||||
|
"no_opt_out_url":
|
||||||
|
f"Opt-Out/Widerspruchs-Link fuer '{name}' fehlt — Art. 21 DSGVO.",
|
||||||
|
"broken_opt_out":
|
||||||
|
f"Opt-Out-Link von '{name}' nicht erreichbar.",
|
||||||
|
"no_name":
|
||||||
|
"Anbieter ohne Name erfasst — Art. 13 Abs. 1 lit. a.",
|
||||||
|
"no_purpose":
|
||||||
|
f"Verarbeitungszweck fuer '{name}' fehlt — Art. 13 Abs. 1 lit. c.",
|
||||||
|
}
|
||||||
|
return hints.get(flag, f"Flag: {flag}")
|
||||||
|
|
||||||
|
|
||||||
|
def _from_redundancies(check_id: str, report: dict | None) -> list[dict]:
|
||||||
|
"""Each redundancy category -> finding row (status='info', sev='LOW')."""
|
||||||
|
if not report:
|
||||||
|
return []
|
||||||
|
out: list[dict] = []
|
||||||
|
for r in (report.get("redundancies") or []):
|
||||||
|
cat = r.get("category_label") or r.get("category") or "Unbekannt"
|
||||||
|
vendors = r.get("vendors") or []
|
||||||
|
sav = r.get("estimated_saving_year_eur") or [0, 0]
|
||||||
|
out.append({
|
||||||
|
"source_type": "redundanz",
|
||||||
|
"doc_type": "-",
|
||||||
|
"severity": "LOW",
|
||||||
|
"status": "info",
|
||||||
|
"regulation": "Cost-Optimization",
|
||||||
|
"label": f"Mehrfach-Anbieter in '{cat}' ({len(vendors)} Tools)",
|
||||||
|
"hint": (
|
||||||
|
f"Anbieter: {', '.join(vendors[:6])}"
|
||||||
|
+ (f" (+{len(vendors)-6} weitere)" if len(vendors) > 6 else "")
|
||||||
|
+ (f" · EU-Empfehlung: {r['suggested_eu_tool']}"
|
||||||
|
if r.get("suggested_eu_tool") else "")
|
||||||
|
),
|
||||||
|
"action_recipe": {
|
||||||
|
"what": "Konsolidierung auf 1 Tool pro Kategorie pruefen.",
|
||||||
|
"why": (r.get("consolidation_hint") or
|
||||||
|
"Mehrfach-Lizenzen + Vertrags-Overhead reduzieren."),
|
||||||
|
"fix_text": "Migrations-Plan zu einem Anbieter erarbeiten; "
|
||||||
|
"Vertraege ueberlappend kuendigen.",
|
||||||
|
},
|
||||||
|
"category": cat,
|
||||||
|
"payload": {
|
||||||
|
"vendors": vendors[:20],
|
||||||
|
"saving_year_eur_low": sav[0],
|
||||||
|
"saving_year_eur_high": sav[1],
|
||||||
|
"suggested_eu_tool": r.get("suggested_eu_tool"),
|
||||||
|
"caveats": (r.get("caveats") or [])[:4],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def collect(
|
||||||
|
check_id: str,
|
||||||
|
results: list[Any],
|
||||||
|
cmp_vendors: list[dict] | None,
|
||||||
|
redundancy_report: dict | None,
|
||||||
|
doc_texts: dict[str, str] | None = None,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Bundle all 4 finding sources into one list ready for record_findings()."""
|
||||||
|
out: list[dict] = []
|
||||||
|
texts = doc_texts or {}
|
||||||
|
for r in (results or []):
|
||||||
|
try:
|
||||||
|
out.extend(_from_doc_check(check_id, r, texts.get(r.doc_type, "")))
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("collect: doc result %s failed: %s",
|
||||||
|
getattr(r, "doc_type", "?"), e)
|
||||||
|
try:
|
||||||
|
out.extend(_from_vendors(check_id, cmp_vendors or []))
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("collect: vendors failed: %s", e)
|
||||||
|
try:
|
||||||
|
out.extend(_from_redundancies(check_id, redundancy_report))
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("collect: redundancies failed: %s", e)
|
||||||
|
logger.info("collect: check=%s total_findings=%d", check_id, len(out))
|
||||||
|
return out
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
"""
|
||||||
|
Unified-Findings sidecar store.
|
||||||
|
|
||||||
|
A compliance check produces findings from 4 sources today:
|
||||||
|
- Master-Controls (mc_results table — already persisted)
|
||||||
|
- Pflichtangaben (L1/L2 doc checks, e.g. Impressum-Vollstaendigkeit)
|
||||||
|
- Vendor scans (per cmp_vendor: missing privacy url, no opt-out, ...)
|
||||||
|
- Redundancies (multi-vendor in same category)
|
||||||
|
|
||||||
|
Previously the DSB had to look in 4 different blocks of the email to
|
||||||
|
find everything. This store flattens all of them into ONE searchable
|
||||||
|
table so the /audit/<check_id> frontend can show a unified list with
|
||||||
|
source / severity / status / doc_type filters.
|
||||||
|
|
||||||
|
Sidecar SQLite (same DB as compliance_audit_log) — no Postgres
|
||||||
|
migration needed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
DB_PATH = os.getenv("COMPLIANCE_AUDIT_DB", "/data/compliance_audits.db")
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_table() -> None:
|
||||||
|
Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.executescript("""
|
||||||
|
CREATE TABLE IF NOT EXISTS unified_findings (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
check_id TEXT NOT NULL,
|
||||||
|
source_type TEXT NOT NULL, -- mc|pflichtangabe|vendor|redundanz
|
||||||
|
doc_type TEXT, -- impressum|dse|cookie|... or '-' for vendor/redundanz
|
||||||
|
severity TEXT, -- CRITICAL|HIGH|MEDIUM|LOW|INFO
|
||||||
|
status TEXT, -- failed|passed|skipped|na|info
|
||||||
|
regulation TEXT,
|
||||||
|
label TEXT,
|
||||||
|
hint TEXT,
|
||||||
|
action_recipe TEXT, -- JSON {what,why,fix_text,where,example}
|
||||||
|
anchor_excerpt TEXT,
|
||||||
|
anchor_conf REAL,
|
||||||
|
vendor_name TEXT,
|
||||||
|
category TEXT,
|
||||||
|
payload TEXT -- JSON extras (matched_text, cookies count, ...)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_uf_check ON unified_findings(check_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_uf_source ON unified_findings(check_id, source_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_uf_status ON unified_findings(check_id, status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_uf_severity ON unified_findings(check_id, severity);
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def record_findings(check_id: str, findings: list[dict]) -> int:
|
||||||
|
"""Bulk-insert all findings for a check. Idempotent on check_id."""
|
||||||
|
if not check_id:
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
_ensure_table()
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM unified_findings WHERE check_id=?", (check_id,),
|
||||||
|
)
|
||||||
|
if not findings:
|
||||||
|
conn.commit()
|
||||||
|
return 0
|
||||||
|
rows = [
|
||||||
|
(
|
||||||
|
check_id,
|
||||||
|
(f.get("source_type") or "mc")[:24],
|
||||||
|
(f.get("doc_type") or "")[:32],
|
||||||
|
(f.get("severity") or "MEDIUM").upper()[:16],
|
||||||
|
(f.get("status") or "failed")[:16],
|
||||||
|
(f.get("regulation") or "")[:64],
|
||||||
|
(f.get("label") or "")[:400],
|
||||||
|
(f.get("hint") or "")[:1200],
|
||||||
|
json.dumps(f.get("action_recipe") or {}, ensure_ascii=False),
|
||||||
|
(f.get("anchor_excerpt") or "")[:800],
|
||||||
|
float(f.get("anchor_conf") or 0.0),
|
||||||
|
(f.get("vendor_name") or "")[:160],
|
||||||
|
(f.get("category") or "")[:64],
|
||||||
|
json.dumps(f.get("payload") or {}, ensure_ascii=False),
|
||||||
|
)
|
||||||
|
for f in findings
|
||||||
|
]
|
||||||
|
conn.executemany(
|
||||||
|
"INSERT INTO unified_findings "
|
||||||
|
"(check_id, source_type, doc_type, severity, status, regulation, "
|
||||||
|
" label, hint, action_recipe, anchor_excerpt, anchor_conf, "
|
||||||
|
" vendor_name, category, payload) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
logger.info(
|
||||||
|
"unified_findings: %s rows=%d sources=%s",
|
||||||
|
check_id, len(rows),
|
||||||
|
sorted(set(f.get("source_type", "mc") for f in findings)),
|
||||||
|
)
|
||||||
|
return len(rows)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("record_findings failed for %s: %s", check_id, e)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def list_findings(
|
||||||
|
check_id: str,
|
||||||
|
source_type: str | None = None,
|
||||||
|
severity: str | None = None,
|
||||||
|
doc_type: str | None = None,
|
||||||
|
status: str | None = None,
|
||||||
|
q: str | None = None,
|
||||||
|
limit: int = 1000,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Return filtered findings. q matches label OR vendor_name (case-insensitive)."""
|
||||||
|
try:
|
||||||
|
_ensure_table()
|
||||||
|
where = ["check_id = ?"]
|
||||||
|
params: list = [check_id]
|
||||||
|
if source_type and source_type != "all":
|
||||||
|
where.append("source_type = ?")
|
||||||
|
params.append(source_type)
|
||||||
|
if severity and severity != "all":
|
||||||
|
where.append("severity = ?")
|
||||||
|
params.append(severity.upper())
|
||||||
|
if doc_type and doc_type != "all":
|
||||||
|
where.append("doc_type = ?")
|
||||||
|
params.append(doc_type)
|
||||||
|
if status and status != "all":
|
||||||
|
where.append("status = ?")
|
||||||
|
params.append(status)
|
||||||
|
if q:
|
||||||
|
where.append("(LOWER(label) LIKE ? OR LOWER(vendor_name) LIKE ?)")
|
||||||
|
needle = f"%{q.lower()}%"
|
||||||
|
params.extend([needle, needle])
|
||||||
|
sql = ("SELECT * FROM unified_findings WHERE " + " AND ".join(where) +
|
||||||
|
" ORDER BY CASE severity "
|
||||||
|
" WHEN 'CRITICAL' THEN 0 WHEN 'HIGH' THEN 1 "
|
||||||
|
" WHEN 'MEDIUM' THEN 2 WHEN 'LOW' THEN 3 "
|
||||||
|
" ELSE 4 END, source_type, label LIMIT ?")
|
||||||
|
params.append(int(limit))
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
rows = conn.execute(sql, params).fetchall()
|
||||||
|
out = []
|
||||||
|
for r in rows:
|
||||||
|
d = dict(r)
|
||||||
|
d["action_recipe"] = json.loads(d.get("action_recipe") or "{}")
|
||||||
|
d["payload"] = json.loads(d.get("payload") or "{}")
|
||||||
|
out.append(d)
|
||||||
|
return out
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("list_findings failed: %s", e)
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def findings_summary(check_id: str) -> dict:
|
||||||
|
"""Return aggregate counts for the filter UI (source/severity/status)."""
|
||||||
|
out = {
|
||||||
|
"total": 0,
|
||||||
|
"by_source": {},
|
||||||
|
"by_severity": {},
|
||||||
|
"by_status": {},
|
||||||
|
"by_doc_type": {},
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
_ensure_table()
|
||||||
|
with sqlite3.connect(DB_PATH) as conn:
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
for col in ("source_type", "severity", "status", "doc_type"):
|
||||||
|
rows = conn.execute(
|
||||||
|
f"SELECT {col} AS k, COUNT(*) AS n FROM unified_findings "
|
||||||
|
f"WHERE check_id=? GROUP BY {col}",
|
||||||
|
(check_id,),
|
||||||
|
).fetchall()
|
||||||
|
bucket = f"by_{col if col != 'source_type' else 'source'}"
|
||||||
|
if col == "doc_type":
|
||||||
|
bucket = "by_doc_type"
|
||||||
|
out[bucket] = {r["k"] or "-": r["n"] for r in rows}
|
||||||
|
out["total"] = max(out["total"], sum(r["n"] for r in rows))
|
||||||
|
return out
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("findings_summary failed: %s", e)
|
||||||
|
return out
|
||||||
@@ -50,6 +50,8 @@ from compliance.api.agent_recurring_routes import router as agent_recurring_rout
|
|||||||
from compliance.api.agent_compare_routes import router as agent_compare_router
|
from compliance.api.agent_compare_routes import router as agent_compare_router
|
||||||
from compliance.api.agent_doc_check_routes import router as agent_doc_check_router
|
from compliance.api.agent_doc_check_routes import router as agent_doc_check_router
|
||||||
from compliance.api.agent_compliance_check_routes import router as agent_compliance_check_router
|
from compliance.api.agent_compliance_check_routes import router as agent_compliance_check_router
|
||||||
|
from compliance.api.agent_findings_routes import router as agent_findings_router
|
||||||
|
from compliance.api.saving_scan_routes import router as saving_scan_router
|
||||||
from compliance.api.agent_migration_routes import router as agent_migration_router
|
from compliance.api.agent_migration_routes import router as agent_migration_router
|
||||||
from compliance.api.vendor_assessment_routes import router as vendor_assessment_router
|
from compliance.api.vendor_assessment_routes import router as vendor_assessment_router
|
||||||
from compliance.api.cra_routes import router as cra_router
|
from compliance.api.cra_routes import router as cra_router
|
||||||
@@ -157,6 +159,8 @@ app.include_router(agent_recurring_router, prefix="/api")
|
|||||||
app.include_router(agent_compare_router, prefix="/api")
|
app.include_router(agent_compare_router, prefix="/api")
|
||||||
app.include_router(agent_doc_check_router, prefix="/api")
|
app.include_router(agent_doc_check_router, prefix="/api")
|
||||||
app.include_router(agent_compliance_check_router, prefix="/api")
|
app.include_router(agent_compliance_check_router, prefix="/api")
|
||||||
|
app.include_router(agent_findings_router, prefix="/api")
|
||||||
|
app.include_router(saving_scan_router, prefix="/api")
|
||||||
app.include_router(agent_migration_router, prefix="/api")
|
app.include_router(agent_migration_router, prefix="/api")
|
||||||
|
|
||||||
# Vendor Contract Assessment
|
# Vendor Contract Assessment
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""
|
||||||
|
Tests for the saving-scan funnel endpoint.
|
||||||
|
|
||||||
|
Focus: input validation + lead persistence + rate-limit error path.
|
||||||
|
The actual compliance check is mocked — we only verify the route layer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||||
|
|
||||||
|
# Use a temp SQLite for the sidecar
|
||||||
|
os.environ["COMPLIANCE_AUDIT_DB"] = "/tmp/test_saving_scan.db"
|
||||||
|
if os.path.exists("/tmp/test_saving_scan.db"):
|
||||||
|
os.remove("/tmp/test_saving_scan.db")
|
||||||
|
|
||||||
|
from compliance.api.saving_scan_routes import router # noqa: E402
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(router, prefix="/api")
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
class TestStartSavingScanValidation:
|
||||||
|
|
||||||
|
def test_missing_email_returns_422(self):
|
||||||
|
resp = client.post("/api/compliance/agent/saving-scan/start",
|
||||||
|
json={"url": "https://example.de"})
|
||||||
|
assert resp.status_code == 422
|
||||||
|
|
||||||
|
def test_invalid_email_returns_400(self):
|
||||||
|
with patch("compliance.api.saving_scan_routes.asyncio.create_task"):
|
||||||
|
resp = client.post(
|
||||||
|
"/api/compliance/agent/saving-scan/start",
|
||||||
|
json={"url": "https://example.de", "email": "kein-email",
|
||||||
|
"consent": True},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert "E-Mail" in resp.json()["detail"]
|
||||||
|
|
||||||
|
def test_invalid_url_returns_400(self):
|
||||||
|
with patch("compliance.api.saving_scan_routes.asyncio.create_task"):
|
||||||
|
resp = client.post(
|
||||||
|
"/api/compliance/agent/saving-scan/start",
|
||||||
|
json={"url": "ftp://wrong.de", "email": "u@x.de",
|
||||||
|
"consent": True},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
def test_consent_required(self):
|
||||||
|
with patch("compliance.api.saving_scan_routes.asyncio.create_task"):
|
||||||
|
resp = client.post(
|
||||||
|
"/api/compliance/agent/saving-scan/start",
|
||||||
|
json={"url": "https://example.de", "email": "u@x.de",
|
||||||
|
"consent": False},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert "Consent" in resp.json()["detail"]
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_check_runner():
|
||||||
|
"""Stub the lazy-imported worker — avoids loading smtp_sender (Py3.10+)."""
|
||||||
|
import sys, types
|
||||||
|
fake = types.ModuleType("compliance.api.agent_compliance_check_routes")
|
||||||
|
|
||||||
|
class _DocInput:
|
||||||
|
def __init__(self, doc_type="other", url=""): self.doc_type, self.url = doc_type, url
|
||||||
|
|
||||||
|
class _Req:
|
||||||
|
def __init__(self, **kw): self.__dict__.update(kw)
|
||||||
|
|
||||||
|
async def _runner(*_a, **_kw): pass
|
||||||
|
|
||||||
|
fake.DocumentInput = _DocInput
|
||||||
|
fake.ComplianceCheckRequest = _Req
|
||||||
|
fake._run_compliance_check = _runner
|
||||||
|
fake._compliance_check_jobs = {}
|
||||||
|
sys.modules["compliance.api.agent_compliance_check_routes"] = fake
|
||||||
|
|
||||||
|
|
||||||
|
class TestStartSavingScanSuccess:
|
||||||
|
|
||||||
|
def test_valid_request_starts_check(self):
|
||||||
|
_patch_check_runner()
|
||||||
|
resp = client.post(
|
||||||
|
"/api/compliance/agent/saving-scan/start",
|
||||||
|
json={"url": "https://example-newdomain.de",
|
||||||
|
"email": "user@example.de", "consent": True},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200, resp.text
|
||||||
|
data = resp.json()
|
||||||
|
assert "check_id" in data
|
||||||
|
assert data["status"] == "running"
|
||||||
|
assert "example-newdomain.de" in data["message"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestLeadCount:
|
||||||
|
|
||||||
|
def test_lead_count_after_submit(self):
|
||||||
|
_patch_check_runner()
|
||||||
|
client.post(
|
||||||
|
"/api/compliance/agent/saving-scan/start",
|
||||||
|
json={"url": "https://abc-leadtest.de",
|
||||||
|
"email": "lead@x.de", "consent": True},
|
||||||
|
)
|
||||||
|
resp = client.get("/api/compliance/agent/saving-scan/lead-count")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
data = resp.json()
|
||||||
|
assert data["total_leads"] >= 1
|
||||||
|
assert "abc-leadtest.de" in str(data["top_domains"])
|
||||||
Reference in New Issue
Block a user