perf(audit): parallel Tesseract OCR + Pipeline-Wire-In für Slicing
ocr_slices_extract_cookies nutzt jetzt ThreadPoolExecutor (4 workers). Tesseract released die GIL, daher echtes parallelisieren möglich. Sequenziell 32 slices ≈ 60s, parallel ~15s. Pipeline in agent_compliance_check_routes.py: Step C ruft jetzt capture_cookie_evidence_slices + ocr_slices_extract_cookies. Source 'tesseract_ocr' wird zu existing Vendors gemergt; neue Vendors als eigenständige Records. Final VW-Scan-Resultat: - Cookies: 60 (parse_flat) → 128 (mit Tesseract) = +113% - Vendors: 18 unique - Adobe Analytics: 9 → 33 Cookies (Tesseract fand +24) Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
@@ -948,54 +948,63 @@ async def _run_compliance_check(check_id: str, req: ComplianceCheckRequest):
|
||||
except Exception as e:
|
||||
logger.warning("crawled-table-parse failed: %s", e)
|
||||
|
||||
# C — Screenshot + Vision-OCR der Cookie-Richtlinie.
|
||||
# Liefert deterministisch die echte Cookie-Tabelle aus dem
|
||||
# gerenderten DOM (Banner akzeptiert, Accordions ausgeklappt,
|
||||
# Timestamp eingebrannt). Komplementaer zu parse_flat: wenn
|
||||
# parse_flat versagt (textContent ohne Whitespace, ungewohntes
|
||||
# Spalten-Layout, andere Sprache), greift die Vision-Extraktion
|
||||
# immer noch zu — sie liest die Tabelle wie ein Mensch.
|
||||
# C — Screenshot + Tesseract-OCR der Cookie-Richtlinie.
|
||||
# Overlapping scrolling screenshots (jede Slice ueberlappt die
|
||||
# vorherige um overlap_px Pixel) → lueckenlose Beweiskette.
|
||||
# Pro Slice Tesseract OCR + parse_ocr_cookie_table; Dedup nach
|
||||
# Cookie-Name über alle Slices. Site-unabhaengig, deterministisch.
|
||||
cookie_url_for_shot = ""
|
||||
for _e in doc_entries:
|
||||
if _e.get("doc_type") == "cookie" and _e.get("url"):
|
||||
cookie_url_for_shot = _e["url"]; break
|
||||
cookie_evidence_screenshot: dict | None = None
|
||||
cookie_evidence_slices: list[dict] | None = None
|
||||
cookie_evidence_meta: dict | None = None
|
||||
if cookie_url_for_shot:
|
||||
try:
|
||||
from compliance.services.cookie_screenshot_ocr import (
|
||||
capture_cookie_screenshot,
|
||||
extract_cookies_via_vision,
|
||||
capture_cookie_evidence_slices,
|
||||
ocr_slices_extract_cookies,
|
||||
cookies_to_vendor_records,
|
||||
)
|
||||
from compliance.services.cookies_table_parser import (
|
||||
_guess_vendor as _gv,
|
||||
)
|
||||
_update(check_id,
|
||||
"Cookie-Tabelle wird fotografiert + OCR-extrahiert...",
|
||||
93)
|
||||
cap = await capture_cookie_screenshot(
|
||||
"Cookie-Richtlinie wird fotografiert (lueckenlose Beweiskette)...",
|
||||
92)
|
||||
ev = await capture_cookie_evidence_slices(
|
||||
cookie_url_for_shot, check_id=check_id,
|
||||
viewport_h=1024, overlap_px=200, max_slices=40,
|
||||
)
|
||||
if cap.get("png_b64"):
|
||||
cookie_evidence_screenshot = cap # fuer ZIP-Anhang
|
||||
vis_cookies = await extract_cookies_via_vision(
|
||||
cap["png_b64"],
|
||||
if ev.get("slices"):
|
||||
cookie_evidence_slices = ev["slices"] # ZIP-Anhang
|
||||
cookie_evidence_meta = {
|
||||
"total_height_px": ev.get("total_height_px"),
|
||||
"width_px": ev.get("width_px"),
|
||||
"accepted_banner": ev.get("accepted_banner"),
|
||||
"expanded": ev.get("expanded"),
|
||||
"url": ev.get("url"),
|
||||
"slice_count": len(ev["slices"]),
|
||||
}
|
||||
_update(check_id,
|
||||
"Tesseract OCR über alle Slices...", 93)
|
||||
ocr_cookies, ocr_stats = ocr_slices_extract_cookies(
|
||||
ev["slices"],
|
||||
)
|
||||
if vis_cookies:
|
||||
vis_vendors = cookies_to_vendor_records(
|
||||
vis_cookies, guess_vendor_fn=_gv,
|
||||
if ocr_cookies:
|
||||
ocr_vendors = cookies_to_vendor_records(
|
||||
ocr_cookies, guess_vendor_fn=_gv,
|
||||
)
|
||||
existing = {
|
||||
(v.get("name") or "").strip().lower()
|
||||
for v in cmp_vendors
|
||||
}
|
||||
added_v = 0
|
||||
for v in vis_vendors:
|
||||
for v in ocr_vendors:
|
||||
nm = (v.get("name") or "").strip()
|
||||
if not nm:
|
||||
continue
|
||||
if nm.lower() in existing:
|
||||
# merge cookies into existing record
|
||||
for ex in cmp_vendors:
|
||||
if (ex.get("name") or "").strip().lower() == nm.lower():
|
||||
ex_names = {
|
||||
@@ -1007,21 +1016,22 @@ async def _run_compliance_check(check_id: str, req: ComplianceCheckRequest):
|
||||
ex.setdefault("cookies", []).append(c)
|
||||
ex_names.add(c["name"].lower())
|
||||
cur_src = ex.get("source", "")
|
||||
if "vision_ocr" not in cur_src:
|
||||
ex["source"] = (cur_src + ";vision_ocr").strip(";")
|
||||
if "tesseract_ocr" not in cur_src:
|
||||
ex["source"] = (cur_src + ";tesseract_ocr").strip(";")
|
||||
break
|
||||
continue
|
||||
cmp_vendors.append(v)
|
||||
existing.add(nm.lower())
|
||||
added_v += 1
|
||||
logger.info(
|
||||
"C Vision-OCR: +%d Vendors / %d Cookies "
|
||||
"(total: %d)",
|
||||
added_v, len(vis_cookies), len(cmp_vendors),
|
||||
"C Tesseract-OCR: +%d Vendors / %d Cookies "
|
||||
"(über %d Slices, total: %d)",
|
||||
added_v, len(ocr_cookies),
|
||||
ocr_stats.get("slices", 0), len(cmp_vendors),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Vision-OCR pipeline failed: %s (%s)",
|
||||
"Tesseract-OCR pipeline failed: %s (%s)",
|
||||
str(e) or "(no msg)", type(e).__name__,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user