breakpilot-compliance/backend-compliance/compliance/api/agent_scan_routes.py

"""
Agent Website Scan Routes — deep scan endpoint that performs multi-page
website analysis with SOLL/IST service comparison.

POST /api/compliance/agent/scan
"""

import logging
import os
import re
from datetime import datetime, timezone

import httpx
from fastapi import APIRouter
from pydantic import BaseModel

from compliance.services.website_scanner import scan_website, DetectedService
from compliance.services.dse_service_extractor import extract_dse_services, compare_services
from compliance.services.smtp_sender import send_email
from compliance.services.dse_parser import parse_dse
from compliance.services.dse_matcher import build_text_references, TextReference
from compliance.services.mandatory_content_checker import (
    check_mandatory_documents, check_dse_mandatory_content, MandatoryFinding,
)
from compliance.services.legal_basis_validator import validate_legal_bases
from compliance.api.agent_scan_helpers import add_corrections, build_scan_summary

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/compliance/agent", tags=["agent"])

SDK_URL = os.environ.get("AI_SDK_URL", "http://bp-compliance-ai-sdk:8090")
TENANT_ID = "9282a473-5c95-4b3a-bf78-0ecc0ec71d3e"
USER_ID = "00000000-0000-0000-0000-000000000001"
SDK_HEADERS = {
    "Content-Type": "application/json",
    "X-Tenant-ID": TENANT_ID,
    "X-User-ID": USER_ID,
}


class ScanRequest(BaseModel):
    url: str
    mode: str = "post_launch"
    recipient: str = "dsb@breakpilot.local"


class ServiceInfo(BaseModel):
    name: str
    category: str
    provider: str
    country: str
    eu_adequate: bool
    requires_consent: bool
    legal_ref: str
    in_dse: bool
    status: str  # "ok", "undocumented", "outdated"


class TextReferenceModel(BaseModel):
    found: bool = False
    source_url: str = ""
    document_type: str = "Datenschutzerklaerung"
    section_heading: str = ""
    section_number: str = ""
    parent_section: str = ""
    paragraph_index: int = 0
    original_text: str = ""
    issue: str = ""
    correction_type: str = ""
    correction_text: str = ""
    insert_after: str = ""


class ScanFinding(BaseModel):
    code: str
    severity: str
    text: str
    correction: str = ""
    text_reference: TextReferenceModel | None = None


class DiscoveredDocument(BaseModel):
    title: str
    url: str
    doc_type: str
    language: str = ""
    word_count: int = 0
    completeness_pct: int = 0
    findings_count: int = 0


class ScanResponse(BaseModel):
    url: str
    pages_scanned: int
    pages_list: list[str] = []
    services: list[ServiceInfo]
    findings: list[ScanFinding]
    discovered_documents: list[DiscoveredDocument] = []
    ai_detected: bool
    chatbot_detected: bool
    chatbot_provider: str
    missing_pages: dict
    summary: str
    email_status: str
    scanned_at: str


@router.post("/scan", response_model=ScanResponse)
async def scan_website_endpoint(req: ScanRequest):
    """Deep website scan: multi-page crawl + SOLL/IST service comparison."""
    is_live = req.mode == "post_launch"

    # Step 1: Scan website — try Playwright first (JS-rendered), fallback to httpx
    playwright_htmls: dict[str, str] = {}
    try:
        async with httpx.AsyncClient(timeout=120.0) as pw_client:
            pw_resp = await pw_client.post(
                "http://bp-compliance-consent-tester:8094/website-scan",
                json={"url": req.url, "max_pages": 15, "click_nav": True},
            )
            if pw_resp.status_code == 200:
                pw_data = pw_resp.json()
                playwright_htmls = pw_data.get("page_htmls", {})
                logger.info("Playwright scan: %d pages, %d scripts",
                            pw_data.get("pages_count", 0), len(pw_data.get("external_scripts", [])))
    except Exception as e:
        logger.warning("Playwright scanner unavailable, falling back to httpx: %s", e)

    # Use Playwright results if available, otherwise fall back to httpx scanner
    if playwright_htmls:
        # Build ScanResult from Playwright data
        from compliance.services.website_scanner import ScanResult, DetectedService, _detect_services, _detect_ai_mentions
        from compliance.services.service_registry import SERVICE_REGISTRY
        scan = ScanResult()
        scan.pages_scanned = list(playwright_htmls.keys())
        for page_url, html in playwright_htmls.items():
            _detect_services(html, page_url, scan)
            _detect_ai_mentions(html, page_url, scan)
        # Deduplicate
        seen = set()
        unique = []
        for svc in scan.detected_services:
            if svc.id not in seen:
                seen.add(svc.id)
                unique.append(svc)
        scan.detected_services = unique
        scan.chatbot_detected = any(s.category == "chatbot" for s in scan.detected_services)
        if scan.chatbot_detected:
            scan.chatbot_provider = next(s.name for s in scan.detected_services if s.category == "chatbot")
    else:
        scan = await scan_website(req.url)

    logger.info("Scanned %d pages, found %d services", len(scan.pages_scanned), len(scan.detected_services))

    # Step 1b: DSI Discovery — find all legal documents on the website
    discovered_docs: list[DiscoveredDocument] = []
    dsi_findings: list[ScanFinding] = []
    try:
        async with httpx.AsyncClient(timeout=180.0) as dsi_client:
            dsi_resp = await dsi_client.post(
                "http://bp-compliance-consent-tester:8094/dsi-discovery",
                json={"url": req.url, "max_documents": 20},
            )
            if dsi_resp.status_code == 200:
                dsi_data = dsi_resp.json()
                logger.info("DSI discovery: %d documents found", dsi_data.get("total_found", 0))

                # Check each document against its legal requirements
                from compliance.services.dsi_document_checker import (
                    check_document_completeness, classify_document_type,
                )
                for doc in dsi_data.get("documents", []):
                    doc_type = classify_document_type(doc["title"], doc["url"])
                    doc_findings = check_document_completeness(
                        doc.get("text_preview", ""), doc_type, doc["title"], doc["url"],
                    )
                    # Count completeness
                    score_finding = next((f for f in doc_findings if "SCORE" in f.get("code", "")), None)
                    completeness = 0
                    if score_finding:
                        import re as _re2
                        pct_match = _re2.search(r"(\d+)%", score_finding.get("text", ""))
                        if pct_match:
                            completeness = int(pct_match.group(1))

                    discovered_docs.append(DiscoveredDocument(
                        title=doc["title"], url=doc["url"],
                        doc_type=doc_type, language=doc.get("language", ""),
                        word_count=doc.get("word_count", 0),
                        completeness_pct=completeness,
                        findings_count=len([f for f in doc_findings if "SCORE" not in f.get("code", "")]),
                    ))
                    for df in doc_findings:
                        if "SCORE" not in df.get("code", ""):
                            dsi_findings.append(ScanFinding(
                                code=df["code"], severity=df["severity"], text=df["text"],
                            ))
    except Exception as e:
        logger.warning("DSI discovery failed: %s", e)

    # Step 2: Fetch privacy policy text (from Playwright HTMLs or httpx)
    dse_text = ""
    for page_url, html in playwright_htmls.items():
        if re.search(r"datenschutz|privacy|dsgvo", page_url, re.IGNORECASE):
            import re as _re
            clean = _re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=_re.DOTALL | _re.IGNORECASE)
            clean = _re.sub(r"<[^>]+>", " ", clean)
            clean = _re.sub(r"\s+", " ", clean).strip()
            dse_text = clean[:4000]
            break
    if not dse_text:
        dse_text = await _fetch_dse_text(req.url, scan.pages_scanned)

    # Step 3: Extract services mentioned in DSE via LLM + text fallback
    dse_services = await extract_dse_services(dse_text) if dse_text else []
    logger.info("DSE mentions %d services (LLM)", len(dse_services))

    # Fallback: if LLM extraction failed, search DSE text directly for service names
    if not dse_services and dse_text:
        dse_lower = dse_text.lower()
        detected_dicts_for_check = [_service_to_dict(s) for s in scan.detected_services]
        for svc in detected_dicts_for_check:
            name = svc.get("name", "").lower()
            # Check if service name appears in DSE text
            if name and len(name) > 3 and name in dse_lower:
                dse_services.append({"name": svc["name"], "purpose": "", "country": svc.get("country", ""), "legal_basis": ""})
        if dse_services:
            logger.info("DSE text fallback found %d services", len(dse_services))

    # Step 4: Parse DSE into structured sections (prefer Playwright HTML)
    dse_html = ""
    for page_url, html in playwright_htmls.items():
        if re.search(r"datenschutz|privacy|dsgvo", page_url, re.IGNORECASE):
            dse_html = html
            break
    if not dse_html:
        dse_html = await _fetch_dse_html(req.url, scan.pages_scanned)
    dse_sections = parse_dse(dse_html, req.url) if dse_html else []
    logger.info("Parsed %d DSE sections", len(dse_sections))

    # Step 5: SOLL/IST comparison
    detected_dicts = [_service_to_dict(s) for s in scan.detected_services]
    comparison = compare_services(detected_dicts, dse_services)

    # Step 6: Build TextReferences for each detected service
    text_refs = build_text_references(detected_dicts, dse_services, dse_sections, req.url)

    # Step 7: Generate findings with text references
    services_info, findings = _build_findings(comparison, scan, is_live, text_refs)

    # Step 8: Check mandatory content (documents + DSE sections)
    mandatory_findings = check_mandatory_documents(scan.pages_scanned, scan.missing_pages)
    mandatory_findings += check_dse_mandatory_content(dse_sections, dse_text)
    for mf in mandatory_findings:
        findings.append(ScanFinding(
            code=mf.code, severity=mf.severity,
            text=f"{mf.text}" + (f" — {mf.suggestion}" if mf.suggestion else ""),
        ))

    # Step 8b: Validate legal bases (lit. a-f) in DSE
    if dse_text:
        lit_findings = validate_legal_bases(dse_text)
        for lf in lit_findings:
            findings.append(ScanFinding(
                code=f"LIT-{lf.purpose.upper()}",
                severity=lf.severity,
                text=lf.text,
                text_reference=TextReferenceModel(
                    found=True, source_url=req.url,
                    original_text=lf.original_text,
                    issue="incorrect", correction_type="replace",
                    correction_text=f"Korrekte Rechtsgrundlage: {lf.correct_basis} ({lf.legal_ref})",
                ) if lf.original_text else None,
            ))

    # Step 8c: Add DSI document findings
    findings.extend(dsi_findings)

    # Step 9: Generate corrections for pre-launch mode
    if not is_live and findings:
        await add_corrections(findings, dse_text)

    # Step 7: Build summary
    summary = build_scan_summary(req.url, scan, comparison, findings, is_live, discovered_docs)

    # Step 8: Send notification
    mode_label = "INTERNE PRUEFUNG" if not is_live else "LIVE-WEBSITE"
    email_result = send_email(
        recipient=req.recipient,
        subject=f"[{mode_label}] Website-Scan: {req.url[:50]}",
        body_html=f"<pre>{summary}</pre>",
    )

    return ScanResponse(
        url=req.url,
        pages_scanned=len(scan.pages_scanned),
        pages_list=scan.pages_scanned,
        services=services_info,
        findings=findings,
        discovered_documents=discovered_docs,
        ai_detected=len(scan.ai_mentions) > 0,
        chatbot_detected=scan.chatbot_detected,
        chatbot_provider=scan.chatbot_provider,
        missing_pages=scan.missing_pages,
        summary=summary,
        email_status=email_result.get("status", "failed"),
        scanned_at=datetime.now(timezone.utc).isoformat(),
    )


async def _fetch_dse_text(url: str, scanned_pages: list[str]) -> str:
    """Find and fetch the privacy policy page text."""
    import re
    # Find DSE URL from scanned pages
    dse_url = None
    for page in scanned_pages:
        if re.search(r"datenschutz|privacy|dsgvo", page, re.IGNORECASE):
            dse_url = page
            break
    if not dse_url:
        dse_url = url  # Fallback to provided URL

    try:
        async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
            resp = await client.get(dse_url, headers={"User-Agent": "BreakPilot-Compliance-Agent/1.0"})
            html = resp.text
            clean = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
            clean = re.sub(r"<[^>]+>", " ", clean)
            clean = re.sub(r"\s+", " ", clean).strip()
            return clean[:4000]
    except Exception:
        return ""


async def _fetch_dse_html(url: str, scanned_pages: list[str]) -> str:
    """Fetch the raw HTML of the privacy policy page (for structured parsing)."""
    import re
    dse_url = None
    for page in scanned_pages:
        if re.search(r"datenschutz|privacy|dsgvo", page, re.IGNORECASE):
            dse_url = page
            break
    if not dse_url:
        dse_url = url
    try:
        async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
            resp = await client.get(dse_url, headers={"User-Agent": "BreakPilot-Compliance-Agent/1.0"})
            return resp.text
    except Exception:
        return ""


def _service_to_dict(svc: DetectedService) -> dict:
    return {
        "id": svc.id, "name": svc.name, "category": svc.category,
        "provider": svc.provider, "country": svc.country,
        "eu_adequate": svc.eu_adequate, "requires_consent": svc.requires_consent,
        "legal_ref": svc.legal_ref,
    }


def _build_findings(
    comparison: dict, scan, is_live: bool, text_refs: dict | None = None,
) -> tuple[list[ServiceInfo], list[ScanFinding]]:
    """Build service info list and findings from comparison."""
    services = []
    findings = []
    text_refs = text_refs or {}

    def _get_ref(svc_id: str) -> TextReferenceModel | None:
        ref = text_refs.get(svc_id)
        if not ref:
            return None
        return TextReferenceModel(
            found=ref.found, source_url=ref.source_url,
            document_type=ref.document_type, section_heading=ref.section_heading,
            section_number=ref.section_number, parent_section=ref.parent_section,
            paragraph_index=ref.paragraph_index, original_text=ref.original_text,
            issue=ref.issue, correction_type=ref.correction_type,
            correction_text=ref.correction_text, insert_after=ref.insert_after,
        )

    # Undocumented services (on website, NOT in DSE)
    for svc in comparison["undocumented"]:
        services.append(ServiceInfo(
            name=svc["name"], category=svc.get("category", "other"),
            provider=svc.get("provider", ""), country=svc.get("country", ""),
            eu_adequate=svc.get("eu_adequate", False),
            requires_consent=svc.get("requires_consent", False),
            legal_ref=svc.get("legal_ref", ""), in_dse=False, status="undocumented",
        ))
        severity = "HIGH" if is_live else "MEDIUM"
        ref = _get_ref(svc.get("id", ""))
        findings.append(ScanFinding(
            code=f"DSE-MISSING-{svc['id'].upper()}",
            severity=severity,
            text=f"{svc['name']} ({svc.get('provider', '')}, {svc.get('country', '')}) "
                 f"ist auf der Website eingebunden aber NICHT in der Datenschutzerklaerung "
                 f"dokumentiert (Art. 13 DSGVO).",
            text_reference=ref,
        ))

    # Documented services (OK)
    for item in comparison["documented"]:
        svc = item["detected"]
        services.append(ServiceInfo(
            name=svc["name"], category=svc.get("category", "other"),
            provider=svc.get("provider", ""), country=svc.get("country", ""),
            eu_adequate=svc.get("eu_adequate", False),
            requires_consent=svc.get("requires_consent", False),
            legal_ref=svc.get("legal_ref", ""), in_dse=True, status="ok",
        ))
        # Check third-country transfer
        if not svc.get("eu_adequate", False):
            findings.append(ScanFinding(
                code=f"TRANSFER-{svc['id'].upper()}",
                severity="MEDIUM",
                text=f"{svc['name']} ({svc.get('country', '')}) — Drittlandtransfer. "
                     f"Pruefen ob SCCs oder Angemessenheitsbeschluss dokumentiert sind.",
            ))

    # Outdated services (in DSE, NOT on website)
    for svc in comparison["outdated"]:
        services.append(ServiceInfo(
            name=svc["name"], category="other",
            provider=svc.get("provider", ""), country=svc.get("country", ""),
            eu_adequate=True, requires_consent=False,
            legal_ref="", in_dse=True, status="outdated",
        ))
        findings.append(ScanFinding(
            code=f"DSE-OUTDATED-{svc['name'].upper().replace(' ', '_')[:20]}",
            severity="LOW",
            text=f"{svc['name']} in Datenschutzerklaerung erwaehnt aber auf der Website "
                 f"nicht mehr gefunden. Eintrag bei naechster Aktualisierung entfernen.",
        ))

    # Missing pages (e.g., /impressum returns 404)
    for page_url, status_code in scan.missing_pages.items():
        if "impressum" in page_url.lower():
            findings.append(ScanFinding(
                code="MISSING-IMPRESSUM",
                severity="HIGH",
                text=f"Impressum-Seite gibt HTTP {status_code} zurueck (§5 TMG Verstoss).",
            ))

    return services, findings