"""Phase D-1 — Vendor-extraction raw stages.

Covers (in the original Step 5 of `_run_compliance_check`):
  - Aggregate cmp_payloads from all doc_entries + banner_result (P30/P48)
  - Fallback: use DSE text when cookie was deduped (P17-D)
  - Extract structured vendor records from CMP payloads
  - LLM-cascade fallback when structured extract yields < 5 vendors (P52)
  - Phase-G vendor-details append (P57)
  - HTML-table DOM parse (Stage D)
  - Crawled cookie-table parse (Stage B)
  - Tesseract OCR over evidence slices (Stage C) — also captures the
    cookie_evidence_slices used by A1 e-mail attachment
"""

from __future__ import annotations

import logging

from ._helpers import _company_name_from_url, _update

logger = logging.getLogger(__name__)


async def run_phase_d1(state: dict) -> None:
    """Vendor-extract raw stages. Mutates state in place."""
    check_id = state["check_id"]
    doc_entries = state["doc_entries"]
    doc_texts = state["doc_texts"]
    banner_result = state["banner_result"]
    pasted_table_vendors = state["pasted_table_vendors"]

    cmp_vendors: list[dict] = []
    cookie_payloads: list[dict] = []
    cookie_text = ""
    cookie_evidence_slices: list[dict] | None = None
    cookie_evidence_meta: dict | None = None

    try:
        from compliance.services.vendor_extractor import (
            extract_vendors_from_payloads,
        )

        # P30: aggregate cmp_payloads from ALL doc_entries — sites
        # like Mercedes load Usercentrics only on the homepage, so the
        # JSON gets captured during DSE/Impressum discovery, not in the
        # cookies.html fetch. Dedup by URL since the same payload is
        # captured on every page load.
        seen_cmp_urls: set[str] = set()
        for e in doc_entries:
            for p in (e.get("cmp_payloads") or []):
                p_url = p.get("url") or ""
                if p_url and p_url in seen_cmp_urls:
                    continue
                seen_cmp_urls.add(p_url)
                cookie_payloads.append(p)
            if e.get("doc_type") == "cookie" and e.get("text"):
                cookie_text = e["text"]
        # P48: also pull cmp_payloads from the Banner-Scan (homepage 3-phase
        # consent test). Mercedes' Usercentrics-JSON is captured there even
        # when not in DSI-Discovery of static legal pages.
        if banner_result:
            for p in (banner_result.get("cmp_payloads") or []):
                p_url = p.get("url") or ""
                if p_url and p_url in seen_cmp_urls:
                    continue
                seen_cmp_urls.add(p_url)
                cookie_payloads.append(p)
            if cookie_payloads:
                logger.info("P48: %d CMP-payloads available for vendor-extract "
                            "(after Banner-Scan merge)", len(cookie_payloads))
        # P17-D: Fallback wenn cookie via P15 deduped wurde — nutze DSE-Text
        # sofern Cookie-Begriffe drin sind, damit LLM-Vendor-Extract trotzdem
        # greifen kann.
        if not cookie_text and not cookie_payloads:
            dse_t = doc_texts.get("dse", "")
            if dse_t and any(w in dse_t.lower() for w in
                              ("cookie", "tracking", "google analytics", "consent")):
                cookie_text = dse_t
                logger.info("P17-D: vendor-extract Fallback auf DSE (Cookie deduped)")
        owner_name = _company_name_from_url(doc_entries) or ""
        if cookie_payloads:
            cmp_vendors = extract_vendors_from_payloads(
                cookie_payloads, owner_name=owner_name,
            )

        # P52: LLM-Fallback nicht nur wenn 0 Vendors, sondern auch wenn die
        # strukturierten Quellen < 5 Vendors lieferten und der Cookie-Text
        # substantiell ist.
        if (len(cmp_vendors) < 5
                and cookie_text and len(cookie_text.split()) >= 500):
            from compliance.services.vendor_llm_extractor import (
                extract_vendors_via_llm,
            )
            from compliance.services.vendor_classifier import classify
            _update(check_id, "Vendor-Liste per LLM extrahieren...", 94)
            llm_vendors = await extract_vendors_via_llm(cookie_text)
            existing_names = {(v.get("name") or "").strip().lower()
                              for v in cmp_vendors}
            added_llm = 0
            for v in llm_vendors:
                nm = (v.get("name") or "").strip()
                if not nm or nm.lower() in existing_names:
                    continue
                v["recipient_type"] = classify(
                    vendor_name=nm,
                    category=v.get("category", ""),
                    owner_name=owner_name,
                )
                v.setdefault("source", "llm_cascade")
                cmp_vendors.append(v)
                existing_names.add(nm.lower())
                added_llm += 1
            if added_llm:
                logger.info("P52 LLM-Cascade: +%d Vendors (total: %d)",
                            added_llm, len(cmp_vendors))

        # P57: Phase G vendor_details als zusätzliche Vendor-Quelle.
        if banner_result:
            vd_list = banner_result.get("vendor_details") or []
            vd_list = [v for v in vd_list if v.get("name") != "__TDM_OPTOUT__"]
            existing_names = {(v.get("name") or "").strip().lower()
                              for v in cmp_vendors}
            added = 0
            for d in vd_list:
                n = (d.get("name") or "").strip()
                if not n or n.lower() in existing_names:
                    continue
                if n.lower() in ("technisch erforderlich", "analyse und statistik",
                                 "marketing", "alles auswählen",
                                 "alles auswaehlen"):
                    continue
                from compliance.services.vendor_classifier import classify
                cmp_vendors.append({
                    "name": n,
                    "country": "",
                    "purpose": d.get("description", "")[:500],
                    "category": "",
                    "opt_out_url": d.get("opt_out_url", ""),
                    "privacy_policy_url": d.get("privacy_url", ""),
                    "persistence": d.get("retention", ""),
                    "cookies": d.get("cookies", []),
                    "processing_company": d.get("processing_company", ""),
                    "address": d.get("address", ""),
                    "purposes": d.get("purposes", []),
                    "technologies": d.get("technologies", []),
                    "recipient_type": classify(
                        vendor_name=n, category="", owner_name=owner_name,
                    ),
                })
                existing_names.add(n.lower())
                added += 1
            if added:
                logger.info("P57: added %d new vendors from Phase G (total: %d)",
                            added, len(cmp_vendors))

        # D — HTML-Tabellen aus DOM
        for pl in (cookie_payloads or []):
            if pl.get("kind") != "html_table":
                continue
            rows = pl.get("rows") or []
            if len(rows) < 3:
                continue
            try:
                from compliance.services.cookies_table_parser import (
                    parse_cookie_table as _parse_ct_d,
                )
                table_text = "\n".join(rows)
                d_vendors = _parse_ct_d(table_text)
                if d_vendors:
                    existing_d = {(v.get("name") or "").strip().lower()
                                  for v in cmp_vendors}
                    added_d = 0
                    for v in d_vendors:
                        nm = (v.get("name") or "").strip()
                        if not nm or nm.lower() in existing_d:
                            continue
                        v.setdefault("source", "html_table_dom")
                        cmp_vendors.append(v)
                        existing_d.add(nm.lower())
                        added_d += 1
                    if added_d:
                        logger.info("D HTML-Table-DOM-Parse: +%d Vendors aus "
                                    "%d-Zeilen-Tabelle (total: %d)",
                                    added_d, len(rows), len(cmp_vendors))
            except Exception as e:
                logger.warning("html_table parse failed: %s", e)

        # B — cookies_table_parser auch auf gecrawltem Cookie-Text
        if cookie_text and len(cookie_text) >= 500:
            try:
                from compliance.services.cookies_table_parser import (
                    parse_cookie_table as _parse_ct,
                    parse_flat_cookie_text as _parse_flat,
                )
                crawled_table_vendors = _parse_ct(cookie_text)
                if not crawled_table_vendors:
                    crawled_table_vendors = _parse_flat(cookie_text)
                if crawled_table_vendors:
                    existing = {(v.get("name") or "").strip().lower()
                                for v in cmp_vendors}
                    added_c = 0
                    for v in crawled_table_vendors:
                        nm = (v.get("name") or "").strip()
                        if not nm or nm.lower() in existing:
                            continue
                        v.setdefault("source", "table_crawled")
                        cmp_vendors.append(v)
                        existing.add(nm.lower())
                        added_c += 1
                    if added_c:
                        logger.info("B Crawled-Tabellen-Parse: +%d Vendors "
                                    "(total: %d)", added_c, len(cmp_vendors))
            except Exception as e:
                logger.warning("crawled-table-parse failed: %s", e)

        # C — Screenshot + Tesseract-OCR (auch Quelle für A1 ZIP-Anhang)
        cookie_url_for_shot = ""
        for _e in doc_entries:
            if _e.get("doc_type") == "cookie" and _e.get("url"):
                cookie_url_for_shot = _e["url"]; break
        if cookie_url_for_shot:
            try:
                from compliance.services.cookie_screenshot_ocr import (
                    capture_cookie_evidence_slices,
                    cookies_to_vendor_records,
                    ocr_slices_extract_cookies,
                )
                from compliance.services.cookies_table_parser import (
                    _guess_vendor as _gv,
                )
                _update(check_id,
                        "Cookie-Richtlinie wird fotografiert "
                        "(lueckenlose Beweiskette)...", 92)
                ev = await capture_cookie_evidence_slices(
                    cookie_url_for_shot, check_id=check_id,
                    viewport_h=1024, overlap_px=200, max_slices=40,
                )
                if ev.get("slices"):
                    cookie_evidence_slices = ev["slices"]
                    cookie_evidence_meta = {
                        "total_height_px": ev.get("total_height_px"),
                        "width_px": ev.get("width_px"),
                        "accepted_banner": ev.get("accepted_banner"),
                        "expanded": ev.get("expanded"),
                        "url": ev.get("url"),
                        "slice_count": len(ev["slices"]),
                    }
                    _update(check_id, "Tesseract OCR über alle Slices...", 93)
                    ocr_cookies, ocr_stats = ocr_slices_extract_cookies(
                        ev["slices"],
                    )
                    if ocr_cookies:
                        ocr_vendors = cookies_to_vendor_records(
                            ocr_cookies, guess_vendor_fn=_gv,
                        )
                        existing = {(v.get("name") or "").strip().lower()
                                    for v in cmp_vendors}
                        added_v = 0
                        for v in ocr_vendors:
                            nm = (v.get("name") or "").strip()
                            if not nm:
                                continue
                            if nm.lower() in existing:
                                for ex in cmp_vendors:
                                    if (ex.get("name") or "").strip().lower() == nm.lower():
                                        ex_names = {
                                            (c.get("name") or "").lower()
                                            for c in (ex.get("cookies") or [])
                                        }
                                        for c in (v.get("cookies") or []):
                                            if c["name"].lower() not in ex_names:
                                                ex.setdefault("cookies", []).append(c)
                                                ex_names.add(c["name"].lower())
                                        cur_src = ex.get("source", "")
                                        if "tesseract_ocr" not in cur_src:
                                            ex["source"] = (cur_src + ";tesseract_ocr").strip(";")
                                        break
                                continue
                            cmp_vendors.append(v)
                            existing.add(nm.lower())
                            added_v += 1
                        logger.info(
                            "C Tesseract-OCR: +%d Vendors / %d Cookies "
                            "(über %d Slices, total: %d)",
                            added_v, len(ocr_cookies),
                            ocr_stats.get("slices", 0), len(cmp_vendors),
                        )
            except Exception as e:
                logger.warning("Tesseract-OCR pipeline failed: %s (%s)",
                               str(e) or "(no msg)", type(e).__name__)

        # User-pasted Cookie-Tabelle (deterministisch, kein LLM):
        # die hat IMMER Vorrang weil 100% genau.
        if pasted_table_vendors:
            existing = {(v.get("name") or "").strip().lower()
                        for v in cmp_vendors}
            added_p = 0
            for v in pasted_table_vendors:
                nm = (v.get("name") or "").strip()
                if not nm or nm.lower() in existing:
                    continue
                cmp_vendors.append(v)
                existing.add(nm.lower())
                added_p += 1
            if added_p:
                logger.info("Pasted-Tabellen-Merge: +%d Vendors (total: %d)",
                            added_p, len(cmp_vendors))
    except Exception as e:
        logger.warning("VVT vendor extraction skipped: %s", e)

    state["cmp_vendors"] = cmp_vendors
    state["cookie_payloads"] = cookie_payloads
    state["cookie_text"] = cookie_text
    state["cookie_evidence_slices"] = cookie_evidence_slices
    state["cookie_evidence_meta"] = cookie_evidence_meta
