breakpilot-compliance/backend-compliance/compliance/api/agent_doc_check_routes.py

"""
Agent Document Check Routes — Multi-URL document verification.

The user provides explicit URLs + document types. No crawling needed.
Each document is loaded, expanded (accordions/tabs), text extracted,
and checked against its type-specific legal checklist.

POST /api/compliance/agent/doc-check
"""

import asyncio
import logging
import os
import uuid as _uuid
from datetime import datetime, timezone

import httpx
from fastapi import APIRouter
from pydantic import BaseModel

from compliance.services.dsi_document_checker import (
    check_document_completeness, classify_document_type,
)
from compliance.services.smtp_sender import send_email

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/compliance/agent", tags=["agent"])

CONSENT_TESTER_URL = "http://bp-compliance-consent-tester:8094"


class DocCheckEntry(BaseModel):
    doc_type: str  # dse, agb, impressum, cookie, widerruf, other
    label: str
    url: str


class DocCheckRequest(BaseModel):
    entries: list[DocCheckEntry]
    recipient: str = "dsb@breakpilot.local"
    check_cookie_banner: bool = False


class CheckItem(BaseModel):
    id: str
    label: str
    passed: bool
    severity: str
    matched_text: str = ""


class DocCheckResult(BaseModel):
    label: str
    url: str
    doc_type: str
    word_count: int = 0
    completeness_pct: int = 0
    checks: list[CheckItem] = []
    findings_count: int = 0
    error: str = ""


class DocCheckResponse(BaseModel):
    results: list[DocCheckResult]
    cookie_banner_result: dict | None = None
    total_documents: int
    total_findings: int
    email_status: str = ""
    checked_at: str


# In-memory job store for async processing
_doc_check_jobs: dict[str, dict] = {}


class DocCheckStartResponse(BaseModel):
    check_id: str
    status: str = "running"


class DocCheckStatusResponse(BaseModel):
    check_id: str
    status: str
    progress: str = ""
    result: DocCheckResponse | None = None
    error: str = ""


@router.post("/doc-check")
async def start_doc_check(req: DocCheckRequest):
    """Start async multi-URL document check."""
    check_id = str(_uuid.uuid4())[:8]
    _doc_check_jobs[check_id] = {"status": "running", "progress": "Pruefung gestartet...", "result": None, "error": ""}
    asyncio.create_task(_run_doc_check(check_id, req))
    return DocCheckStartResponse(check_id=check_id, status="running")


@router.get("/doc-check/{check_id}")
async def get_doc_check_status(check_id: str):
    """Poll document check status."""
    job = _doc_check_jobs.get(check_id)
    if not job:
        return {"check_id": check_id, "status": "not_found"}
    return DocCheckStatusResponse(
        check_id=check_id, status=job["status"],
        progress=job.get("progress", ""), result=job.get("result"),
        error=job.get("error", ""),
    )


async def _run_doc_check(check_id: str, req: DocCheckRequest):
    """Background task: check each document."""
    try:
        results: list[DocCheckResult] = []
        total_findings = 0

        for i, entry in enumerate(req.entries):
            _doc_check_jobs[check_id]["progress"] = (
                f"Dokument {i+1}/{len(req.entries)}: {entry.label}..."
            )

            result = await _check_single_document(entry)
            results.append(result)
            total_findings += result.findings_count

        # Optional: Cookie banner check on first URL
        cookie_result = None
        if req.check_cookie_banner and req.entries:
            _doc_check_jobs[check_id]["progress"] = "Cookie-Banner wird geprueft..."
            cookie_result = await _check_cookie_banner(req.entries[0].url)

        # Build email report
        _doc_check_jobs[check_id]["progress"] = "Report wird erstellt..."
        summary = _build_report(results, cookie_result)
        email_result = send_email(
            recipient=req.recipient,
            subject=f"[DOKUMENTEN-PRUEFUNG] {len(results)} Dokumente geprueft",
            body_html=f"<pre>{summary}</pre>",
        )

        response = DocCheckResponse(
            results=results,
            cookie_banner_result=cookie_result,
            total_documents=len(results),
            total_findings=total_findings,
            email_status=email_result.get("status", "failed"),
            checked_at=datetime.now(timezone.utc).isoformat(),
        )

        _doc_check_jobs[check_id]["status"] = "completed"
        _doc_check_jobs[check_id]["result"] = response
        _doc_check_jobs[check_id]["progress"] = "Fertig"

    except Exception as e:
        logger.error("Doc check %s failed: %s", check_id, e)
        _doc_check_jobs[check_id]["status"] = "failed"
        _doc_check_jobs[check_id]["error"] = str(e)[:500]


async def _check_single_document(entry: DocCheckEntry) -> DocCheckResult:
    """Load a single URL, expand content, extract text, run checklist."""
    try:
        async with httpx.AsyncClient(timeout=90.0) as client:
            resp = await client.post(
                f"{CONSENT_TESTER_URL}/dsi-discovery",
                json={"url": entry.url, "max_documents": 1},
            )
            if resp.status_code != 200:
                return DocCheckResult(
                    label=entry.label, url=entry.url, doc_type=entry.doc_type,
                    error=f"Seite nicht erreichbar (HTTP {resp.status_code})",
                )

            data = resp.json()
            docs = data.get("documents", [])

            # Use the first document found, or fall back to any text
            doc_text = ""
            word_count = 0
            if docs:
                doc_text = docs[0].get("full_text", "") or docs[0].get("text_preview", "")
                word_count = docs[0].get("word_count", 0)

            if not doc_text or len(doc_text) < 50:
                return DocCheckResult(
                    label=entry.label, url=entry.url, doc_type=entry.doc_type,
                    error="Kein Text extrahierbar",
                )

            # Run checklist
            findings = check_document_completeness(
                doc_text, entry.doc_type, entry.label, entry.url,
            )

            # Extract all_checks from SCORE finding
            all_checks: list[CheckItem] = []
            completeness = 0
            for f in findings:
                if "SCORE" in f.get("code", ""):
                    checks_data = f.get("all_checks", [])
                    all_checks = [
                        CheckItem(
                            id=c["id"], label=c["label"], passed=c["passed"],
                            severity=c["severity"], matched_text=c.get("matched_text", ""),
                        )
                        for c in checks_data
                    ]
                    # Extract percentage
                    import re
                    pct_match = re.search(r"(\d+)%", f.get("text", ""))
                    if pct_match:
                        completeness = int(pct_match.group(1))

            non_score = [f for f in findings if "SCORE" not in f.get("code", "")]

            return DocCheckResult(
                label=entry.label, url=entry.url, doc_type=entry.doc_type,
                word_count=word_count, completeness_pct=completeness,
                checks=all_checks, findings_count=len(non_score),
            )

    except Exception as e:
        logger.warning("Doc check failed for %s: %s", entry.url, e)
        return DocCheckResult(
            label=entry.label, url=entry.url, doc_type=entry.doc_type,
            error=str(e)[:200],
        )


async def _check_cookie_banner(url: str) -> dict | None:
    """Run cookie banner consent test on a URL."""
    try:
        async with httpx.AsyncClient(timeout=120.0) as client:
            resp = await client.post(
                f"{CONSENT_TESTER_URL}/scan",
                json={"url": url, "timeout_per_phase": 8},
            )
            if resp.status_code == 200:
                return resp.json()
    except Exception as e:
        logger.warning("Cookie banner check failed: %s", e)
    return None


def _build_report(results: list[DocCheckResult], cookie_result: dict | None) -> str:
    """Build email report."""
    parts = [
        "DOKUMENTEN-PRUEFUNG",
        f"Dokumente geprueft: {len(results)}",
        "",
    ]
    for r in results:
        status = "OK" if r.completeness_pct == 100 else "LUECKENHAFT" if r.completeness_pct >= 50 else "MANGELHAFT"
        if r.error:
            status = "FEHLER"
        parts.append(f"[{status}] {r.label} ({r.completeness_pct}%, {r.word_count} Woerter)")

        for check in r.checks:
            icon = "+" if check.passed else "!!"
            parts.append(f"  [{icon}] {check.label}")

        if r.error:
            parts.append(f"  FEHLER: {r.error}")
        parts.append("")

    if cookie_result:
        parts.extend([
            "Cookie-Banner Pruefung:",
            f"  Banner erkannt: {cookie_result.get('banner_detected', False)}",
            f"  Anbieter: {cookie_result.get('banner_provider', 'unbekannt')}",
        ])
        violations = cookie_result.get("banner_checks", {}).get("violations", [])
        if violations:
            for v in violations[:10]:
                parts.append(f"  [!!] {v.get('text', '')[:80]}")
        else:
            parts.append("  Keine Verstoesse erkannt.")

    return "\n".join(parts)