"""
FastAPI routes for the Control Generator Pipeline.

Endpoints:
  POST /v1/canonical/generate                     — Start generation run
  GET  /v1/canonical/generate/status/{job_id}     — Job status
  GET  /v1/canonical/generate/jobs                — All jobs
  GET  /v1/canonical/generate/review-queue        — Controls needing review
  POST /v1/canonical/generate/review/{control_id} — Complete review
  GET  /v1/canonical/generate/processed-stats     — Processing stats per collection
  GET  /v1/canonical/blocked-sources              — Blocked sources list
  POST /v1/canonical/blocked-sources/cleanup      — Start cleanup workflow
"""

import asyncio
import json
import logging
from typing import Optional, List

from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
from sqlalchemy import text

from db.session import SessionLocal
from services.control_generator import (
    ControlGeneratorPipeline,
    GeneratorConfig,
    ALL_COLLECTIONS,
    VALID_CATEGORIES,
    VALID_DOMAINS,
    _classify_regulation,
    _detect_category,
    _detect_domain,
    _llm_local,
    _parse_llm_json,
    CATEGORY_LIST_STR,
)
from services.citation_backfill import CitationBackfill, BackfillResult
from services.rag_client import get_rag_client
from services.anchor_finder import AnchorFinder, OpenAnchor
from services.control_generator import GeneratedControl as _GeneratedControl

logger = logging.getLogger(__name__)
router = APIRouter(prefix="/v1/canonical", tags=["control-generator"])


# =============================================================================
# REQUEST / RESPONSE MODELS
# =============================================================================

class GenerateRequest(BaseModel):
    domain: Optional[str] = None
    collections: Optional[List[str]] = None
    max_controls: int = 50
    max_chunks: int = 1000  # Default: process max 1000 chunks per job (respects document boundaries)
    batch_size: int = 5
    skip_web_search: bool = True  # Default True — Anchors nachtraeglich batchen
    dry_run: bool = False
    regulation_filter: Optional[List[str]] = None  # Only process these regulation_code prefixes
    regulation_exclude: Optional[List[str]] = None  # Skip these regulation_code prefixes
    skip_prefilter: bool = False  # Skip local LLM pre-filter, send all chunks to API


class GenerateResponse(BaseModel):
    job_id: str
    status: str
    message: str
    total_chunks_scanned: int = 0
    controls_generated: int = 0
    controls_verified: int = 0
    controls_needs_review: int = 0
    controls_too_close: int = 0
    controls_duplicates_found: int = 0
    controls_qa_fixed: int = 0
    errors: list = []
    controls: list = []


class ReviewRequest(BaseModel):
    action: str  # "approve", "reject", "needs_rework"
    release_state: Optional[str] = None  # Override release_state
    notes: Optional[str] = None


class ProcessedStats(BaseModel):
    collection: str
    total_chunks_estimated: int
    processed_chunks: int
    pending_chunks: int
    direct_adopted: int
    llm_reformed: int
    skipped: int


class BlockedSourceResponse(BaseModel):
    id: str
    regulation_code: str
    document_title: str
    reason: str
    deletion_status: str
    qdrant_collection: Optional[str] = None
    marked_at: str


# =============================================================================
# ENDPOINTS
# =============================================================================

async def _run_pipeline_background(config: GeneratorConfig, job_id: str):
    """Run the pipeline in the background. Uses its own DB session."""
    db = SessionLocal()
    try:
        config.existing_job_id = job_id
        pipeline = ControlGeneratorPipeline(db=db, rag_client=get_rag_client())
        result = await pipeline.run(config)
        logger.info(
            "Background generation job %s completed: %d controls from %d chunks",
            job_id, result.controls_generated, result.total_chunks_scanned,
        )
    except Exception as e:
        logger.error("Background generation job %s failed: %s", job_id, e)
        # Update job as failed
        try:
            db.execute(
                text("""
                    UPDATE canonical_generation_jobs
                    SET status = 'failed', errors = :errors, completed_at = NOW()
                    WHERE id = CAST(:job_id AS uuid)
                """),
                {"job_id": job_id, "errors": json.dumps([str(e)])},
            )
            db.commit()
        except Exception:
            pass
    finally:
        db.close()


@router.post("/generate", response_model=GenerateResponse)
async def start_generation(req: GenerateRequest):
    """Start a control generation run (runs in background).

    Returns immediately with job_id. Use GET /generate/status/{job_id} to poll progress.
    """
    config = GeneratorConfig(
        collections=req.collections,
        domain=req.domain,
        batch_size=req.batch_size,
        max_controls=req.max_controls,
        max_chunks=req.max_chunks,
        skip_web_search=req.skip_web_search,
        dry_run=req.dry_run,
        regulation_filter=req.regulation_filter,
        regulation_exclude=req.regulation_exclude,
        skip_prefilter=req.skip_prefilter,
    )

    if req.dry_run:
        # Dry run: execute synchronously and return controls
        db = SessionLocal()
        try:
            pipeline = ControlGeneratorPipeline(db=db, rag_client=get_rag_client())
            result = await pipeline.run(config)
            return GenerateResponse(
                job_id=result.job_id,
                status=result.status,
                message=f"Dry run: {result.controls_generated} controls from {result.total_chunks_scanned} chunks",
                total_chunks_scanned=result.total_chunks_scanned,
                controls_generated=result.controls_generated,
                controls_verified=result.controls_verified,
                controls_needs_review=result.controls_needs_review,
                controls_too_close=result.controls_too_close,
                controls_duplicates_found=result.controls_duplicates_found,
                errors=result.errors,
                controls=result.controls,
            )
        except Exception as e:
            logger.error("Dry run failed: %s", e)
            raise HTTPException(status_code=500, detail=str(e))
        finally:
            db.close()

    # Create job record first so we can return the ID
    db = SessionLocal()
    try:
        result = db.execute(
            text("""
                INSERT INTO canonical_generation_jobs (status, config)
                VALUES ('running', :config)
                RETURNING id
            """),
            {"config": json.dumps(config.model_dump())},
        )
        db.commit()
        row = result.fetchone()
        job_id = str(row[0]) if row else None
    except Exception as e:
        logger.error("Failed to create job: %s", e)
        raise HTTPException(status_code=500, detail=f"Failed to create job: {e}")
    finally:
        db.close()

    if not job_id:
        raise HTTPException(status_code=500, detail="Failed to create job record")

    # Launch pipeline in background
    asyncio.create_task(_run_pipeline_background(config, job_id))

    return GenerateResponse(
        job_id=job_id,
        status="running",
        message="Generation started in background. Poll /generate/status/{job_id} for progress.",
    )


@router.get("/generate/status/{job_id}")
async def get_job_status(job_id: str):
    """Get status of a generation job."""
    db = SessionLocal()
    try:
        result = db.execute(
            text("SELECT * FROM canonical_generation_jobs WHERE id = CAST(:id AS uuid)"),
            {"id": job_id},
        )
        row = result.fetchone()
        if not row:
            raise HTTPException(status_code=404, detail="Job not found")

        cols = result.keys()
        job = dict(zip(cols, row))
        # Serialize datetime fields
        for key in ("started_at", "completed_at", "created_at"):
            if job.get(key):
                job[key] = str(job[key])
        job["id"] = str(job["id"])
        return job
    finally:
        db.close()


@router.get("/generate/jobs")
async def list_jobs(
    limit: int = Query(20, ge=1, le=100),
    offset: int = Query(0, ge=0),
):
    """List all generation jobs."""
    db = SessionLocal()
    try:
        result = db.execute(
            text("""
                SELECT id, status, total_chunks_scanned, controls_generated,
                       controls_verified, controls_needs_review, controls_too_close,
                       controls_duplicates_found, created_at, completed_at
                FROM canonical_generation_jobs
                ORDER BY created_at DESC
                LIMIT :limit OFFSET :offset
            """),
            {"limit": limit, "offset": offset},
        )
        jobs = []
        cols = result.keys()
        for row in result:
            job = dict(zip(cols, row))
            job["id"] = str(job["id"])
            for key in ("created_at", "completed_at"):
                if job.get(key):
                    job[key] = str(job[key])
            jobs.append(job)
        return {"jobs": jobs, "total": len(jobs)}
    finally:
        db.close()


@router.get("/generate/review-queue")
async def get_review_queue(
    release_state: str = Query("needs_review", pattern="^(needs_review|too_close|duplicate)$"),
    limit: int = Query(50, ge=1, le=200),
):
    """Get controls that need manual review."""
    db = SessionLocal()
    try:
        result = db.execute(
            text("""
                SELECT c.id, c.control_id, c.title, c.objective, c.severity,
                       c.release_state, c.license_rule, c.customer_visible,
                       c.generation_metadata, c.open_anchors, c.tags,
                       c.created_at
                FROM canonical_controls c
                WHERE c.release_state = :state
                ORDER BY c.created_at DESC
                LIMIT :limit
            """),
            {"state": release_state, "limit": limit},
        )
        controls = []
        cols = result.keys()
        for row in result:
            ctrl = dict(zip(cols, row))
            ctrl["id"] = str(ctrl["id"])
            ctrl["created_at"] = str(ctrl["created_at"])
            # Parse JSON fields
            for jf in ("generation_metadata", "open_anchors", "tags"):
                if isinstance(ctrl.get(jf), str):
                    try:
                        ctrl[jf] = json.loads(ctrl[jf])
                    except (json.JSONDecodeError, TypeError):
                        pass
            controls.append(ctrl)
        return {"controls": controls, "total": len(controls)}
    finally:
        db.close()


@router.post("/generate/review/{control_id}")
async def review_control(control_id: str, req: ReviewRequest):
    """Complete review of a generated control."""
    db = SessionLocal()
    try:
        # Validate control exists and is in reviewable state
        result = db.execute(
            text("SELECT id, release_state FROM canonical_controls WHERE control_id = :cid"),
            {"cid": control_id},
        )
        row = result.fetchone()
        if not row:
            raise HTTPException(status_code=404, detail="Control not found")

        current_state = row[1]
        if current_state not in ("needs_review", "too_close", "duplicate"):
            raise HTTPException(status_code=400, detail=f"Control is in state '{current_state}', not reviewable")

        # Determine new state
        if req.action == "approve":
            new_state = req.release_state or "draft"
        elif req.action == "reject":
            new_state = "deprecated"
        elif req.action == "needs_rework":
            new_state = "needs_review"
        else:
            raise HTTPException(status_code=400, detail=f"Unknown action: {req.action}")

        if new_state not in ("draft", "review", "approved", "deprecated", "needs_review", "too_close", "duplicate"):
            raise HTTPException(status_code=400, detail=f"Invalid release_state: {new_state}")

        db.execute(
            text("""
                UPDATE canonical_controls
                SET release_state = :state, updated_at = NOW()
                WHERE control_id = :cid
            """),
            {"state": new_state, "cid": control_id},
        )
        db.commit()

        return {"control_id": control_id, "release_state": new_state, "action": req.action}
    finally:
        db.close()


class BulkReviewRequest(BaseModel):
    release_state: str  # Filter: which controls to bulk-review
    action: str  # "approve" or "reject"
    new_state: Optional[str] = None  # Override target state


@router.post("/generate/bulk-review")
async def bulk_review(req: BulkReviewRequest):
    """Bulk review all controls matching a release_state filter.

    Example: reject all needs_review → sets them to deprecated.
    """
    if req.release_state not in ("needs_review", "too_close", "duplicate"):
        raise HTTPException(status_code=400, detail=f"Invalid filter state: {req.release_state}")

    if req.action == "approve":
        target = req.new_state or "draft"
    elif req.action == "reject":
        target = "deprecated"
    else:
        raise HTTPException(status_code=400, detail=f"Unknown action: {req.action}")

    if target not in ("draft", "review", "approved", "deprecated", "needs_review"):
        raise HTTPException(status_code=400, detail=f"Invalid target state: {target}")

    db = SessionLocal()
    try:
        result = db.execute(
            text("""
                UPDATE canonical_controls
                SET release_state = :target, updated_at = NOW()
                WHERE release_state = :source
                RETURNING control_id
            """),
            {"source": req.release_state, "target": target},
        )
        affected = [row[0] for row in result]
        db.commit()

        return {
            "action": req.action,
            "source_state": req.release_state,
            "target_state": target,
            "affected_count": len(affected),
        }
    finally:
        db.close()


class QAReclassifyRequest(BaseModel):
    limit: int = 100  # How many controls to reclassify per run
    dry_run: bool = True  # Preview only by default
    filter_category: Optional[str] = None  # Only reclassify controls of this category
    filter_domain_prefix: Optional[str] = None  # Only reclassify controls with this prefix


@router.post("/generate/qa-reclassify")
async def qa_reclassify(req: QAReclassifyRequest):
    """Run QA reclassification on existing controls using local LLM.

    Finds controls where keyword-detection disagrees with current category/domain,
    then uses Ollama to determine the correct classification.
    """
    db = SessionLocal()
    try:
        # Load controls to check
        where_clauses = ["release_state NOT IN ('deprecated')"]
        params = {"limit": req.limit}
        if req.filter_category:
            where_clauses.append("category = :cat")
            params["cat"] = req.filter_category
        if req.filter_domain_prefix:
            where_clauses.append("control_id LIKE :prefix")
            params["prefix"] = f"{req.filter_domain_prefix}-%"

        where_sql = " AND ".join(where_clauses)
        rows = db.execute(
            text(f"""
                SELECT id, control_id, title, objective, category,
                       COALESCE(requirements::text, '[]') as requirements,
                       COALESCE(source_original_text, '') as source_text
                FROM canonical_controls
                WHERE {where_sql}
                ORDER BY created_at DESC
                LIMIT :limit
            """),
            params,
        ).fetchall()

        results = {"checked": 0, "mismatches": 0, "fixes": [], "errors": []}

        for row in rows:
            results["checked"] += 1
            control_id = row[1]
            title = row[2]
            objective = row[3] or ""
            current_category = row[4]
            source_text = row[6] or objective

            # Keyword detection on source text
            kw_category = _detect_category(source_text) or _detect_category(objective)
            kw_domain = _detect_domain(source_text)
            current_prefix = control_id.split("-")[0] if "-" in control_id else ""

            # Skip if keyword detection agrees with current classification
            if kw_category == current_category and kw_domain == current_prefix:
                continue

            results["mismatches"] += 1

            # Ask Ollama to arbitrate
            try:
                reqs_text = ""
                try:
                    reqs = json.loads(row[5])
                    if isinstance(reqs, list):
                        reqs_text = ", ".join(str(r) for r in reqs[:3])
                except Exception:
                    pass

                prompt = f"""Pruefe dieses Compliance-Control auf korrekte Klassifizierung.

Titel: {title[:100]}
Ziel: {objective[:200]}
Anforderungen: {reqs_text[:200]}

Aktuelle Zuordnung: domain={current_prefix}, category={current_category}
Keyword-Erkennung: domain={kw_domain}, category={kw_category}

Welche Zuordnung ist korrekt? Antworte NUR als JSON:
{{"domain": "KUERZEL", "category": "kategorie_name", "reason": "kurze Begruendung"}}

Domains: AUTH=Authentifizierung, CRYP=Kryptographie, NET=Netzwerk, DATA=Datenschutz, LOG=Logging, ACC=Zugriffskontrolle, SEC=IT-Sicherheit, INC=Vorfallmanagement, AI=KI, COMP=Compliance, GOV=Behoerden, LAB=Arbeitsrecht, FIN=Finanzregulierung, TRD=Gewerbe, ENV=Umwelt, HLT=Gesundheit
Kategorien: {CATEGORY_LIST_STR}"""

                raw = await _llm_local(prompt)
                data = _parse_llm_json(raw)
                if not data:
                    continue

                qa_domain = data.get("domain", "").upper()
                qa_category = data.get("category", "")
                reason = data.get("reason", "")

                fix_entry = {
                    "control_id": control_id,
                    "title": title[:80],
                    "old_category": current_category,
                    "old_domain": current_prefix,
                    "new_category": qa_category if qa_category in VALID_CATEGORIES else current_category,
                    "new_domain": qa_domain if qa_domain in VALID_DOMAINS else current_prefix,
                    "reason": reason,
                }

                category_changed = qa_category in VALID_CATEGORIES and qa_category != current_category

                if category_changed and not req.dry_run:
                    db.execute(
                        text("""
                            UPDATE canonical_controls
                            SET category = :category, updated_at = NOW()
                            WHERE id = :id
                        """),
                        {"id": row[0], "category": qa_category},
                    )
                    fix_entry["applied"] = True
                else:
                    fix_entry["applied"] = False

                results["fixes"].append(fix_entry)

            except Exception as e:
                results["errors"].append({"control_id": control_id, "error": str(e)})

        if not req.dry_run:
            db.commit()

        return results
    finally:
        db.close()


@router.get("/generate/processed-stats")
async def get_processed_stats():
    """Get processing statistics per collection."""
    db = SessionLocal()
    try:
        result = db.execute(
            text("""
                SELECT
                    collection,
                    COUNT(*) as processed_chunks,
                    COUNT(*) FILTER (WHERE processing_path = 'structured') as direct_adopted,
                    COUNT(*) FILTER (WHERE processing_path = 'llm_reform') as llm_reformed,
                    COUNT(*) FILTER (WHERE processing_path = 'skipped') as skipped
                FROM canonical_processed_chunks
                GROUP BY collection
                ORDER BY collection
            """)
        )
        stats = []
        cols = result.keys()
        for row in result:
            stat = dict(zip(cols, row))
            stat["total_chunks_estimated"] = 0  # Would need Qdrant API to get total
            stat["pending_chunks"] = 0
            stats.append(stat)
        return {"stats": stats}
    finally:
        db.close()


# =============================================================================
# BLOCKED SOURCES
# =============================================================================

@router.get("/blocked-sources")
async def list_blocked_sources():
    """List all blocked (Rule 3) sources."""
    db = SessionLocal()
    try:
        result = db.execute(
            text("""
                SELECT id, regulation_code, document_title, reason,
                       deletion_status, qdrant_collection, marked_at
                FROM canonical_blocked_sources
                ORDER BY marked_at DESC
            """)
        )
        sources = []
        cols = result.keys()
        for row in result:
            src = dict(zip(cols, row))
            src["id"] = str(src["id"])
            src["marked_at"] = str(src["marked_at"])
            sources.append(src)
        return {"sources": sources}
    finally:
        db.close()


@router.post("/blocked-sources/cleanup")
async def start_cleanup():
    """Start cleanup workflow for blocked sources.

    This marks all pending blocked sources for deletion.
    Actual RAG chunk deletion and file removal is a separate manual step.
    """
    db = SessionLocal()
    try:
        result = db.execute(
            text("""
                UPDATE canonical_blocked_sources
                SET deletion_status = 'marked_for_deletion'
                WHERE deletion_status = 'pending'
                RETURNING regulation_code
            """)
        )
        marked = [row[0] for row in result]
        db.commit()

        return {
            "status": "marked_for_deletion",
            "marked_count": len(marked),
            "regulation_codes": marked,
            "message": "Sources marked for deletion. Run manual cleanup to remove RAG chunks and files.",
        }
    finally:
        db.close()


# =============================================================================
# CUSTOMER VIEW FILTER
# =============================================================================

@router.get("/controls-customer")
async def get_controls_customer_view(
    severity: Optional[str] = Query(None),
    domain: Optional[str] = Query(None),
):
    """Get controls filtered for customer visibility.

    Rule 3 controls have source_citation and source_original_text hidden.
    generation_metadata is NEVER shown to customers.
    """
    db = SessionLocal()
    try:
        query = """
            SELECT c.id, c.control_id, c.title, c.objective, c.rationale,
                   c.scope, c.requirements, c.test_procedure, c.evidence,
                   c.severity, c.risk_score, c.implementation_effort,
                   c.open_anchors, c.release_state, c.tags,
                   c.license_rule, c.customer_visible,
                   c.source_original_text, c.source_citation,
                   c.created_at, c.updated_at
            FROM canonical_controls c
            WHERE c.release_state IN ('draft', 'approved')
        """
        params: dict = {}

        if severity:
            query += " AND c.severity = :severity"
            params["severity"] = severity
        if domain:
            query += " AND c.control_id LIKE :domain"
            params["domain"] = f"{domain.upper()}-%"

        query += " ORDER BY c.control_id"

        result = db.execute(text(query), params)
        controls = []
        cols = result.keys()
        for row in result:
            ctrl = dict(zip(cols, row))
            ctrl["id"] = str(ctrl["id"])
            for key in ("created_at", "updated_at"):
                if ctrl.get(key):
                    ctrl[key] = str(ctrl[key])

            # Parse JSON fields
            for jf in ("scope", "requirements", "test_procedure", "evidence",
                        "open_anchors", "tags", "source_citation"):
                if isinstance(ctrl.get(jf), str):
                    try:
                        ctrl[jf] = json.loads(ctrl[jf])
                    except (json.JSONDecodeError, TypeError):
                        pass

            # Customer visibility rules:
            # - NEVER show generation_metadata
            # - Rule 3: NEVER show source_citation or source_original_text
            ctrl.pop("generation_metadata", None)
            if not ctrl.get("customer_visible", True):
                ctrl["source_citation"] = None
                ctrl["source_original_text"] = None

            controls.append(ctrl)

        return {"controls": controls, "total": len(controls)}
    finally:
        db.close()


# =============================================================================
# CITATION BACKFILL
# =============================================================================

class BackfillRequest(BaseModel):
    dry_run: bool = True  # Default to dry_run for safety
    limit: int = 0  # 0 = all controls


class BackfillResponse(BaseModel):
    status: str
    total_controls: int = 0
    matched_hash: int = 0
    matched_regex: int = 0
    matched_llm: int = 0
    unmatched: int = 0
    updated: int = 0
    errors: list = []


_backfill_status: dict = {}


async def _run_backfill_background(dry_run: bool, limit: int, backfill_id: str):
    """Run backfill in background with own DB session."""
    db = SessionLocal()
    try:
        backfill = CitationBackfill(db=db, rag_client=get_rag_client())
        result = await backfill.run(dry_run=dry_run, limit=limit)
        _backfill_status[backfill_id] = {
            "status": "completed",
            "total_controls": result.total_controls,
            "matched_hash": result.matched_hash,
            "matched_regex": result.matched_regex,
            "matched_llm": result.matched_llm,
            "unmatched": result.unmatched,
            "updated": result.updated,
            "errors": result.errors[:50],
        }
        logger.info("Backfill %s completed: %d updated", backfill_id, result.updated)
    except Exception as e:
        logger.error("Backfill %s failed: %s", backfill_id, e)
        _backfill_status[backfill_id] = {"status": "failed", "errors": [str(e)]}
    finally:
        db.close()


@router.post("/generate/backfill-citations", response_model=BackfillResponse)
async def start_backfill(req: BackfillRequest):
    """Backfill article/paragraph into existing control source_citations.

    Uses 3-tier matching: hash lookup → regex parse → Ollama LLM.
    Default is dry_run=True (preview only, no DB changes).
    """
    import uuid
    backfill_id = str(uuid.uuid4())[:8]
    _backfill_status[backfill_id] = {"status": "running"}

    # Always run in background (RAG index build takes minutes)
    asyncio.create_task(_run_backfill_background(req.dry_run, req.limit, backfill_id))
    return BackfillResponse(
        status=f"running (id={backfill_id})",
    )


@router.get("/generate/backfill-status/{backfill_id}")
async def get_backfill_status(backfill_id: str):
    """Get status of a backfill job."""
    status = _backfill_status.get(backfill_id)
    if not status:
        raise HTTPException(status_code=404, detail="Backfill job not found")
    return status


# =============================================================================
# DOMAIN + TARGET AUDIENCE BACKFILL
# =============================================================================

class DomainBackfillRequest(BaseModel):
    dry_run: bool = True
    job_id: Optional[str] = None  # Only backfill controls from this job
    limit: int = 0  # 0 = all

_domain_backfill_status: dict = {}


async def _run_domain_backfill(req: DomainBackfillRequest, backfill_id: str):
    """Backfill domain, category, and target_audience for existing controls using Anthropic."""
    import os
    import httpx

    ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")
    ANTHROPIC_MODEL = os.getenv("CONTROL_GEN_ANTHROPIC_MODEL", "claude-sonnet-4-6")

    if not ANTHROPIC_API_KEY:
        _domain_backfill_status[backfill_id] = {
            "status": "failed", "error": "ANTHROPIC_API_KEY not set"
        }
        return

    db = SessionLocal()
    try:
        # Find controls needing backfill
        where_clauses = ["(target_audience IS NULL OR target_audience = '[]' OR target_audience = 'null')"]
        params: dict = {}
        if req.job_id:
            where_clauses.append("generation_metadata->>'job_id' = :job_id")
            params["job_id"] = req.job_id

        query = f"""
            SELECT id, control_id, title, objective, category, source_original_text, tags
            FROM canonical_controls
            WHERE {' AND '.join(where_clauses)}
            ORDER BY control_id
        """
        if req.limit > 0:
            query += f" LIMIT {req.limit}"

        result = db.execute(text(query), params)
        controls = [dict(zip(result.keys(), row)) for row in result]

        total = len(controls)
        updated = 0
        errors = []

        _domain_backfill_status[backfill_id] = {
            "status": "running", "total": total, "updated": 0, "errors": []
        }

        # Process in batches of 10
        BATCH_SIZE = 10
        for batch_start in range(0, total, BATCH_SIZE):
            batch = controls[batch_start:batch_start + BATCH_SIZE]

            entries = []
            for idx, ctrl in enumerate(batch):
                text_for_analysis = ctrl.get("objective") or ctrl.get("title") or ""
                original = ctrl.get("source_original_text") or ""
                if original:
                    text_for_analysis += f"\n\nQuelltext-Auszug: {original[:500]}"
                entries.append(
                    f"--- CONTROL {idx + 1}: {ctrl['control_id']} ---\n"
                    f"Titel: {ctrl.get('title', '')}\n"
                    f"Objective: {text_for_analysis[:800]}\n"
                    f"Tags: {json.dumps(ctrl.get('tags', []))}"
                )

            prompt = f"""Analysiere die folgenden {len(batch)} Controls und bestimme fuer jedes:
1. domain: Das Fachgebiet (AUTH, CRYP, NET, DATA, LOG, ACC, SEC, INC, AI, COMP, GOV, LAB, FIN, TRD, ENV, HLT)
2. category: Die Kategorie (encryption, authentication, network, data_protection, logging, incident, continuity, compliance, supply_chain, physical, personnel, application, system, risk, governance, hardware, identity, public_administration, labor_law, finance, trade_regulation, environmental, health)
3. target_audience: Liste der Zielgruppen (moegliche Werte: "unternehmen", "behoerden", "entwickler", "datenschutzbeauftragte", "geschaeftsfuehrung", "it-abteilung", "rechtsabteilung", "compliance-officer", "personalwesen", "einkauf", "produktion", "vertrieb", "gesundheitswesen", "finanzwesen", "oeffentlicher_dienst")

Antworte mit einem JSON-Array mit {len(batch)} Objekten. Jedes Objekt hat:
- control_index: 1-basierter Index
- domain: Fachgebiet-Kuerzel
- category: Kategorie
- target_audience: Liste der Zielgruppen

{"".join(entries)}"""

            try:
                headers = {
                    "x-api-key": ANTHROPIC_API_KEY,
                    "anthropic-version": "2023-06-01",
                    "content-type": "application/json",
                }
                payload = {
                    "model": ANTHROPIC_MODEL,
                    "max_tokens": 4096,
                    "system": "Du bist ein Compliance-Experte. Klassifiziere Controls nach Fachgebiet und Zielgruppe. Antworte NUR mit validem JSON.",
                    "messages": [{"role": "user", "content": prompt}],
                }

                async with httpx.AsyncClient(timeout=60.0) as client:
                    resp = await client.post(
                        "https://api.anthropic.com/v1/messages",
                        headers=headers,
                        json=payload,
                    )
                    if resp.status_code != 200:
                        errors.append(f"Anthropic API {resp.status_code} at batch {batch_start}")
                        continue

                    raw = resp.json().get("content", [{}])[0].get("text", "")

                # Parse response
                import re
                bracket_match = re.search(r"\[.*\]", raw, re.DOTALL)
                if not bracket_match:
                    errors.append(f"No JSON array in response at batch {batch_start}")
                    continue

                results_list = json.loads(bracket_match.group(0))

                for item in results_list:
                    idx = item.get("control_index", 0) - 1
                    if idx < 0 or idx >= len(batch):
                        continue
                    ctrl = batch[idx]
                    ctrl_id = str(ctrl["id"])

                    new_domain = item.get("domain", "")
                    new_category = item.get("category", "")
                    new_audience = item.get("target_audience", [])

                    if not isinstance(new_audience, list):
                        new_audience = []

                    # Build new control_id from domain if domain changed
                    old_prefix = ctrl["control_id"].split("-")[0] if ctrl["control_id"] else ""
                    new_prefix = new_domain.upper()[:4] if new_domain else old_prefix

                    if not req.dry_run:
                        update_parts = []
                        update_params: dict = {"ctrl_id": ctrl_id}

                        if new_category:
                            update_parts.append("category = :category")
                            update_params["category"] = new_category

                        if new_audience:
                            update_parts.append("target_audience = :target_audience")
                            update_params["target_audience"] = json.dumps(new_audience)

                        # Note: We do NOT rename control_ids here — that would
                        # break references and cause unique constraint violations.

                        if update_parts:
                            update_parts.append("updated_at = NOW()")
                            db.execute(
                                text(f"UPDATE canonical_controls SET {', '.join(update_parts)} WHERE id = CAST(:ctrl_id AS uuid)"),
                                update_params,
                            )
                            updated += 1

                if not req.dry_run:
                    db.commit()

            except Exception as e:
                errors.append(f"Batch {batch_start}: {str(e)}")
                db.rollback()

            _domain_backfill_status[backfill_id] = {
                "status": "running", "total": total, "updated": updated,
                "progress": f"{min(batch_start + BATCH_SIZE, total)}/{total}",
                "errors": errors[-10:],
            }

        _domain_backfill_status[backfill_id] = {
            "status": "completed", "total": total, "updated": updated,
            "errors": errors[-50:],
        }
        logger.info("Domain backfill %s completed: %d/%d updated", backfill_id, updated, total)

    except Exception as e:
        logger.error("Domain backfill %s failed: %s", backfill_id, e)
        _domain_backfill_status[backfill_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/backfill-domain")
async def start_domain_backfill(req: DomainBackfillRequest):
    """Backfill domain, category, and target_audience for controls using Anthropic API.

    Finds controls where target_audience is NULL and enriches them.
    Default is dry_run=True (preview only).
    """
    import uuid
    backfill_id = str(uuid.uuid4())[:8]
    _domain_backfill_status[backfill_id] = {"status": "starting"}
    asyncio.create_task(_run_domain_backfill(req, backfill_id))
    return {"status": "running", "backfill_id": backfill_id,
            "message": f"Domain backfill started. Poll /generate/backfill-status/{backfill_id}"}


@router.get("/generate/domain-backfill-status/{backfill_id}")
async def get_domain_backfill_status(backfill_id: str):
    """Get status of a domain backfill job."""
    status = _domain_backfill_status.get(backfill_id)
    if not status:
        raise HTTPException(status_code=404, detail="Domain backfill job not found")
    return status


# =============================================================================
# APPLICABILITY BACKFILL — Industry, company size, scope conditions
# =============================================================================

class ApplicabilityBackfillRequest(BaseModel):
    dry_run: bool = True
    limit: int = 0  # 0 = all
    batch_size: int = 10


_applicability_backfill_status: dict = {}


async def _run_applicability_backfill(req: ApplicabilityBackfillRequest, backfill_id: str):
    """Backfill applicable_industries, applicable_company_size, scope_conditions."""
    import os
    import httpx

    ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")
    ANTHROPIC_MODEL = os.getenv("CONTROL_GEN_ANTHROPIC_MODEL", "claude-sonnet-4-6")

    if not ANTHROPIC_API_KEY:
        _applicability_backfill_status[backfill_id] = {
            "status": "failed", "error": "ANTHROPIC_API_KEY not set"
        }
        return

    db = SessionLocal()
    try:
        limit_clause = f"LIMIT {req.limit}" if req.limit > 0 else ""
        rows = db.execute(text(f"""
            SELECT id, control_id, title, objective, category,
                   source_citation->>'source' as source_name,
                   source_citation->>'regulation_code' as regulation_code
            FROM compliance.canonical_controls
            WHERE release_state = 'draft'
              AND (applicable_industries IS NULL OR applicable_industries::text = 'null')
            ORDER BY control_id
            {limit_clause}
        """)).fetchall()

        total = len(rows)
        updated = 0
        errors = []

        _applicability_backfill_status[backfill_id] = {
            "status": "running", "total": total, "updated": 0,
            "dry_run": req.dry_run, "errors": [],
        }

        BATCH_SIZE = req.batch_size
        for batch_start in range(0, total, BATCH_SIZE):
            batch = rows[batch_start:batch_start + BATCH_SIZE]

            entries = []
            for idx, row in enumerate(batch):
                entries.append(
                    f"--- CONTROL {idx + 1}: {row.control_id} ---\n"
                    f"Titel: {row.title or ''}\n"
                    f"Objective: {(row.objective or '')[:400]}\n"
                    f"Kategorie: {row.category or ''}\n"
                    f"Quelle: {row.source_name or ''} ({row.regulation_code or ''})"
                )

            prompt = f"""Analysiere die folgenden {len(batch)} Compliance-Controls und bestimme fuer jedes:

1. applicable_industries: Liste der Branchen fuer die dieser Control relevant ist.
   Verwende "all" wenn der Control branchenuebergreifend gilt.
   Moegliche Werte: "all", "Technologie/IT", "Finanzdienstleistungen", "Versicherungen",
   "Gesundheitswesen", "Pharma", "Telekommunikation", "Energie", "Produktion/Industrie",
   "Logistik/Transport", "E-Commerce/Handel", "Oeffentlicher Dienst", "Bildung",
   "Beratung/Consulting", "Immobilien", "Bau", "Automobil", "Maschinenbau",
   "Luft-/Raumfahrt", "Medien/Verlage", "Gastronomie/Hotellerie", "Recht/Kanzlei",
   "Agrar", "Chemie", "Verteidigung"
   Beispiel DSGVO: ["all"], Beispiel TKG: ["Telekommunikation"], Beispiel NIS2: ["Energie", "Gesundheitswesen", "Transport", "Finanzdienstleistungen"]

2. applicable_company_size: Ab welcher Unternehmensgroesse gilt dieser Control?
   Verwende "all" wenn keine Groessenbeschraenkung.
   Moegliche Werte: "all", "micro", "small", "medium", "large", "enterprise"
   Beispiel NIS2 Art.21: ["medium", "large", "enterprise"], Beispiel DSGVO Art.5: ["all"]

3. scope_conditions: Optionale Bedingungen. Null wenn keine besonderen Bedingungen.
   Sonst JSON-Objekt mit requires_any und description.
   Moegliche Signale: "uses_ai", "third_country_transfer", "processes_health_data",
   "processes_minors_data", "automated_decisions", "employee_monitoring",
   "video_surveillance", "financial_data", "is_kritis_operator", "payment_services"
   Beispiel AI Act: {{"requires_any": ["uses_ai"], "description": "Nur bei KI-Einsatz"}}
   Beispiel DSGVO Art.32: null (gilt immer)

Antworte mit einem JSON-Array. Jedes Objekt hat:
- control_index: 1-basierter Index
- applicable_industries: Liste
- applicable_company_size: Liste
- scope_conditions: Objekt oder null

{chr(10).join(entries)}"""

            try:
                headers = {
                    "x-api-key": ANTHROPIC_API_KEY,
                    "anthropic-version": "2023-06-01",
                    "content-type": "application/json",
                }
                payload = {
                    "model": ANTHROPIC_MODEL,
                    "max_tokens": 4096,
                    "system": "Du bist ein Compliance-Experte. Klassifiziere Controls nach Branche, Unternehmensgroesse und Scope-Bedingungen. Antworte NUR mit validem JSON.",
                    "messages": [{"role": "user", "content": prompt}],
                }

                async with httpx.AsyncClient(timeout=120.0) as client:
                    resp = await client.post(
                        "https://api.anthropic.com/v1/messages",
                        headers=headers,
                        json=payload,
                    )

                if resp.status_code != 200:
                    errors.append(f"Batch {batch_start}: API {resp.status_code}")
                    continue

                content = resp.json().get("content", [{}])[0].get("text", "")
                parsed = _parse_llm_json(content)
                if not parsed:
                    errors.append(f"Batch {batch_start}: JSON parse failed")
                    continue

                items = parsed if isinstance(parsed, list) else [parsed]

                for item in items:
                    idx = item.get("control_index", 0) - 1
                    if idx < 0 or idx >= len(batch):
                        continue

                    row = batch[idx]
                    industries = item.get("applicable_industries", ["all"])
                    company_size = item.get("applicable_company_size", ["all"])
                    scope = item.get("scope_conditions")

                    if not req.dry_run:
                        db.execute(text("""
                            UPDATE compliance.canonical_controls
                            SET applicable_industries = CAST(:ind AS jsonb),
                                applicable_company_size = CAST(:size AS jsonb),
                                scope_conditions = CAST(:scope AS jsonb),
                                updated_at = NOW()
                            WHERE id = CAST(:cid AS uuid)
                        """), {
                            "ind": json.dumps(industries),
                            "size": json.dumps(company_size),
                            "scope": json.dumps(scope) if scope else None,
                            "cid": str(row.id),
                        })

                    updated += 1

                if not req.dry_run:
                    db.commit()

            except Exception as e:
                errors.append(f"Batch {batch_start}: {str(e)[:200]}")
                logger.warning("Applicability backfill batch %d error: %s", batch_start, e)
                db.rollback()

            _applicability_backfill_status[backfill_id] = {
                "status": "running", "total": total, "updated": updated,
                "progress": f"{min(batch_start + BATCH_SIZE, total)}/{total}",
                "dry_run": req.dry_run, "errors": errors[-10:],
            }

        _applicability_backfill_status[backfill_id] = {
            "status": "completed", "total": total, "updated": updated,
            "dry_run": req.dry_run, "errors": errors[-50:],
        }
        logger.info("Applicability backfill %s completed: %d/%d updated",
                     backfill_id, updated, total)

    except Exception as e:
        logger.error("Applicability backfill %s failed: %s", backfill_id, e)
        _applicability_backfill_status[backfill_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/backfill-applicability")
async def start_applicability_backfill(req: ApplicabilityBackfillRequest):
    """Backfill applicable_industries, applicable_company_size, scope_conditions
    for controls missing these fields. Uses Anthropic API.
    Default is dry_run=True (preview only).
    """
    import uuid
    backfill_id = str(uuid.uuid4())[:8]
    _applicability_backfill_status[backfill_id] = {"status": "starting"}
    asyncio.create_task(_run_applicability_backfill(req, backfill_id))
    return {
        "status": "running",
        "backfill_id": backfill_id,
        "message": f"Applicability backfill started. Poll /generate/applicability-backfill-status/{backfill_id}",
    }


@router.get("/generate/applicability-backfill-status/{backfill_id}")
async def get_applicability_backfill_status(backfill_id: str):
    """Get status of an applicability backfill job."""
    status = _applicability_backfill_status.get(backfill_id)
    if not status:
        raise HTTPException(status_code=404, detail="Applicability backfill job not found")
    return status


# ---------------------------------------------------------------------------
# Source-Type Backfill — Classify law vs guideline vs standard vs restricted
# ---------------------------------------------------------------------------

class SourceTypeBackfillRequest(BaseModel):
    dry_run: bool = True


_source_type_backfill_status: dict = {}


async def _run_source_type_backfill(dry_run: bool, backfill_id: str):
    """Backfill source_type into source_citation JSONB for all controls."""
    db = SessionLocal()
    try:
        # Find controls with source_citation that lack source_type
        rows = db.execute(text("""
            SELECT control_id, source_citation, generation_metadata
            FROM compliance.canonical_controls
            WHERE source_citation IS NOT NULL
              AND (source_citation->>'source_type' IS NULL
                   OR source_citation->>'source_type' = '')
        """)).fetchall()

        total = len(rows)
        updated = 0
        already_correct = 0
        errors = []

        _source_type_backfill_status[backfill_id] = {
            "status": "running", "total": total, "updated": 0, "dry_run": dry_run,
        }

        for row in rows:
            cid = row[0]
            citation = row[1] if isinstance(row[1], dict) else json.loads(row[1] or "{}")
            metadata = row[2] if isinstance(row[2], dict) else json.loads(row[2] or "{}")

            # Get regulation_code from metadata
            reg_code = metadata.get("source_regulation", "")
            if not reg_code:
                # Try to infer from source name
                errors.append(f"{cid}: no source_regulation in metadata")
                continue

            # Classify
            license_info = _classify_regulation(reg_code)
            source_type = license_info.get("source_type", "restricted")

            # Update citation
            citation["source_type"] = source_type

            if not dry_run:
                db.execute(text("""
                    UPDATE compliance.canonical_controls
                    SET source_citation = :citation
                    WHERE control_id = :cid
                """), {"citation": json.dumps(citation), "cid": cid})
                if updated % 100 == 0:
                    db.commit()
            updated += 1

        if not dry_run:
            db.commit()

        # Count distribution
        dist_query = db.execute(text("""
            SELECT source_citation->>'source_type' as st, COUNT(*)
            FROM compliance.canonical_controls
            WHERE source_citation IS NOT NULL
              AND source_citation->>'source_type' IS NOT NULL
            GROUP BY st
        """)).fetchall() if not dry_run else []

        distribution = {r[0]: r[1] for r in dist_query}

        _source_type_backfill_status[backfill_id] = {
            "status": "completed", "total": total, "updated": updated,
            "dry_run": dry_run, "distribution": distribution,
            "errors": errors[:50],
        }
        logger.info("Source-type backfill %s completed: %d/%d updated (dry_run=%s)",
                     backfill_id, updated, total, dry_run)

    except Exception as e:
        logger.error("Source-type backfill %s failed: %s", backfill_id, e)
        _source_type_backfill_status[backfill_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/backfill-source-type")
async def start_source_type_backfill(req: SourceTypeBackfillRequest):
    """Backfill source_type (law/guideline/standard/restricted) into source_citation JSONB.

    Classifies each control's source as binding law, authority guideline,
    voluntary standard, or restricted norm based on regulation_code.
    Default is dry_run=True (preview only).
    """
    import uuid
    backfill_id = str(uuid.uuid4())[:8]
    _source_type_backfill_status[backfill_id] = {"status": "starting"}
    asyncio.create_task(_run_source_type_backfill(req.dry_run, backfill_id))
    return {
        "status": "running",
        "backfill_id": backfill_id,
        "message": f"Source-type backfill started. Poll /generate/source-type-backfill-status/{backfill_id}",
    }


@router.get("/generate/source-type-backfill-status/{backfill_id}")
async def get_source_type_backfill_status(backfill_id: str):
    """Get status of a source-type backfill job."""
    status = _source_type_backfill_status.get(backfill_id)
    if not status:
        raise HTTPException(status_code=404, detail="Source-type backfill job not found")
    return status


# =============================================================================
# REPAIR BACKFILL — Fix controls with missing title/objective/requirements
# =============================================================================

class RepairBackfillRequest(BaseModel):
    dry_run: bool = True
    limit: int = 0  # 0 = all
    batch_size: int = 10


_repair_backfill_status: dict = {}


async def _run_repair_backfill(req: RepairBackfillRequest, backfill_id: str):
    """Repair controls with missing title, objective, or requirements using Anthropic API."""
    import os
    import httpx

    ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")
    ANTHROPIC_MODEL = os.getenv("CONTROL_GEN_ANTHROPIC_MODEL", "claude-sonnet-4-6")

    if not ANTHROPIC_API_KEY:
        _repair_backfill_status[backfill_id] = {
            "status": "failed", "error": "ANTHROPIC_API_KEY not set"
        }
        return

    db = SessionLocal()
    try:
        # Find controls needing repair: missing title OR missing objective OR missing requirements
        limit_clause = f"LIMIT {req.limit}" if req.limit > 0 else ""
        rows = db.execute(text(f"""
            SELECT id, control_id, title, objective, requirements::text as requirements,
                   source_original_text, tags::text as tags, category
            FROM compliance.canonical_controls
            WHERE release_state = 'draft'
              AND (
                (title IS NULL OR title = 'None' OR title = '')
                OR (objective IS NULL OR objective = 'None' OR objective = '')
                OR (requirements IS NULL OR requirements::text = '[]' OR requirements::text = 'null')
              )
            ORDER BY control_id
            {limit_clause}
        """)).fetchall()

        total = len(rows)
        repaired = 0
        skipped = 0
        errors = []

        _repair_backfill_status[backfill_id] = {
            "status": "running", "total": total, "repaired": 0, "skipped": 0,
            "dry_run": req.dry_run, "errors": [],
        }

        for i in range(0, total, req.batch_size):
            batch = rows[i:i + req.batch_size]

            entries = []
            for idx, row in enumerate(batch):
                # Collect all available context
                available = []
                if row.title and row.title != "None":
                    available.append(f"Titel: {row.title}")
                if row.objective and row.objective != "None":
                    available.append(f"Objective: {row.objective[:500]}")
                if row.requirements and row.requirements not in ("[]", "null", "None"):
                    available.append(f"Requirements: {row.requirements[:500]}")
                if row.source_original_text and len(row.source_original_text) > 20:
                    available.append(f"Quelltext: {row.source_original_text[:800]}")
                if row.category:
                    available.append(f"Kategorie: {row.category}")

                missing = []
                if not row.title or row.title == "None":
                    missing.append("title")
                if not row.objective or row.objective == "None":
                    missing.append("objective")
                if not row.requirements or row.requirements in ("[]", "null", "None"):
                    missing.append("requirements")

                entries.append(
                    f"--- CONTROL {idx + 1}: {row.control_id} ---\n"
                    f"Fehlend: {', '.join(missing)}\n"
                    f"{'chr(10)'.join(available)}\n"
                )

            prompt = f"""Repariere die folgenden {len(batch)} Compliance-Controls. Fuer jedes Control fehlen bestimmte Felder.

Regeln:
- title: Kurzer, praegnanter deutscher Titel (max 80 Zeichen). Erster Buchstabe gross.
- objective: 1-2 Saetze die das Ziel des Controls beschreiben.
- requirements: JSON-Array mit 2-5 konkreten Anforderungen als Strings.
- Nur die fehlenden Felder generieren. Bestehende Felder NICHT aendern.
- Wenn nicht genug Kontext vorhanden ist, schreibe "SKIP" als Wert.

Antworte mit einem JSON-Array. Jedes Objekt hat:
- control_index: 1-basierter Index
- title: (nur wenn fehlend, sonst null)
- objective: (nur wenn fehlend, sonst null)
- requirements: (nur wenn fehlend, sonst null — als JSON-Array von Strings)

{chr(10).join(entries)}"""

            try:
                headers = {
                    "x-api-key": ANTHROPIC_API_KEY,
                    "anthropic-version": "2023-06-01",
                    "content-type": "application/json",
                }
                payload = {
                    "model": ANTHROPIC_MODEL,
                    "max_tokens": 4096,
                    "system": "Du bist ein Compliance-Experte. Repariere unvollstaendige Controls. Antworte NUR mit validem JSON.",
                    "messages": [{"role": "user", "content": prompt}],
                }

                async with httpx.AsyncClient(timeout=120.0) as client:
                    resp = await client.post(
                        "https://api.anthropic.com/v1/messages",
                        headers=headers,
                        json=payload,
                    )

                if resp.status_code != 200:
                    errors.append(f"Batch {i}: API {resp.status_code}")
                    continue

                content = resp.json().get("content", [{}])[0].get("text", "")
                parsed = _parse_llm_json(content)
                if not parsed:
                    errors.append(f"Batch {i}: JSON parse failed")
                    continue

                # Handle single object response
                items = parsed if isinstance(parsed, list) else [parsed]

                for item in items:
                    idx = item.get("control_index", 0) - 1
                    if idx < 0 or idx >= len(batch):
                        continue

                    row = batch[idx]
                    updates = []
                    params = {"cid": str(row.id)}

                    new_title = item.get("title")
                    if new_title and new_title != "SKIP" and (not row.title or row.title == "None"):
                        updates.append("title = :title")
                        params["title"] = new_title

                    new_obj = item.get("objective")
                    if new_obj and new_obj != "SKIP" and (not row.objective or row.objective == "None"):
                        updates.append("objective = :objective")
                        params["objective"] = new_obj

                    new_req = item.get("requirements")
                    if new_req and new_req != "SKIP" and (not row.requirements or row.requirements in ("[]", "null", "None")):
                        if isinstance(new_req, list):
                            updates.append("requirements = CAST(:requirements AS jsonb)")
                            params["requirements"] = json.dumps(new_req)

                    if not updates:
                        skipped += 1
                        continue

                    if not req.dry_run:
                        updates.append("updated_at = NOW()")
                        db.execute(text(f"""
                            UPDATE compliance.canonical_controls
                            SET {', '.join(updates)}
                            WHERE id = CAST(:cid AS uuid)
                        """), params)

                    repaired += 1

                if not req.dry_run:
                    db.commit()

            except Exception as e:
                errors.append(f"Batch {i}: {str(e)[:200]}")
                logger.warning("Repair backfill batch %d error: %s", i, e)
                db.rollback()

            _repair_backfill_status[backfill_id] = {
                "status": "running", "total": total, "repaired": repaired, "skipped": skipped,
                "progress": f"{min(i + req.batch_size, total)}/{total}",
                "dry_run": req.dry_run, "errors": errors[-10:],
            }

        _repair_backfill_status[backfill_id] = {
            "status": "completed", "total": total, "repaired": repaired, "skipped": skipped,
            "dry_run": req.dry_run, "errors": errors[-50:],
        }
        logger.info("Repair backfill %s completed: %d/%d repaired, %d skipped",
                     backfill_id, repaired, total, skipped)

    except Exception as e:
        logger.error("Repair backfill %s failed: %s", backfill_id, e)
        _repair_backfill_status[backfill_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/backfill-repair")
async def start_repair_backfill(req: RepairBackfillRequest):
    """Repair controls with missing title, objective, or requirements using Anthropic API.

    Finds draft controls where title/objective/requirements are missing or empty,
    and generates the missing fields from available context (source text, other fields).
    Default is dry_run=True (preview only, no DB changes).
    """
    import uuid
    backfill_id = str(uuid.uuid4())[:8]
    _repair_backfill_status[backfill_id] = {"status": "starting"}
    asyncio.create_task(_run_repair_backfill(req, backfill_id))
    return {
        "status": "running",
        "backfill_id": backfill_id,
        "message": f"Repair backfill started. Poll /generate/repair-backfill-status/{backfill_id}",
    }


@router.get("/generate/repair-backfill-status/{backfill_id}")
async def get_repair_backfill_status(backfill_id: str):
    """Get status of a repair backfill job."""
    status = _repair_backfill_status.get(backfill_id)
    if not status:
        raise HTTPException(status_code=404, detail="Repair backfill job not found")
    return status


# =============================================================================
# BATCH DEDUP
# =============================================================================

class BatchDedupRequest(BaseModel):
    dry_run: bool = True
    hint_filter: Optional[str] = None  # Only process groups matching this hint prefix
    since: Optional[str] = None  # ISO datetime — scope to controls created at/after this


_batch_dedup_status: dict = {}


async def _run_batch_dedup(req: BatchDedupRequest, dedup_id: str):
    """Run batch dedup in background."""
    from services.batch_dedup_runner import BatchDedupRunner

    db = SessionLocal()
    try:
        runner = BatchDedupRunner(db)
        _batch_dedup_status[dedup_id] = {"status": "running", "phase": "starting"}

        since_dt = None
        if req.since:
            from datetime import datetime
            since_dt = datetime.fromisoformat(req.since.replace("Z", "+00:00"))
        stats = await runner.run(
            dry_run=req.dry_run,
            hint_filter=req.hint_filter,
            since=since_dt,
        )

        _batch_dedup_status[dedup_id] = {
            "status": "completed",
            "dry_run": req.dry_run,
            **stats,
        }
        logger.info("BatchDedup %s completed: %s", dedup_id, stats)

    except Exception as e:
        logger.error("BatchDedup %s failed: %s", dedup_id, e)
        _batch_dedup_status[dedup_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/batch-dedup")
async def start_batch_dedup(req: BatchDedupRequest):
    """Run batch deduplication on Pass 0b atomic controls.

    Phase 1: Intra-group dedup (same merge_group_hint → pick best, link rest)
    Phase 2: Cross-group dedup (embed masters, search Qdrant for similar)

    Default is dry_run=True (preview only, no DB changes).
    """
    import uuid
    dedup_id = str(uuid.uuid4())[:8]
    _batch_dedup_status[dedup_id] = {"status": "starting"}
    asyncio.create_task(_run_batch_dedup(req, dedup_id))
    return {
        "status": "running",
        "dedup_id": dedup_id,
        "message": f"BatchDedup started. Poll /generate/batch-dedup-status/{dedup_id}",
    }


@router.get("/generate/batch-dedup-status/{dedup_id}")
async def get_batch_dedup_status(dedup_id: str):
    """Get status of a batch dedup job."""
    status = _batch_dedup_status.get(dedup_id)
    if not status:
        raise HTTPException(status_code=404, detail="BatchDedup job not found")
    return status


# =============================================================================
# LLM DEDUP REVIEW — Local LLM verifies borderline duplicates
# =============================================================================

class LLMDedupRequest(BaseModel):
    dry_run: bool = True
    limit: int = 100
    min_score: float = 0.85  # Only review entries >= this score
    max_score: float = 0.91  # Only review entries < this score (0.91+ already handled)
    model: str = "qwen3.5:35b-a3b"


_llm_dedup_status: dict = {}


async def _run_llm_dedup(req: LLMDedupRequest, job_id: str):
    """Use local LLM to verify borderline dedup matches."""
    import httpx
    import os

    OLLAMA_URL = os.getenv("OLLAMA_URL", "http://host.docker.internal:11434")

    db = SessionLocal()
    try:
        # Load review entries in the score band
        limit_clause = f"LIMIT {req.limit}" if req.limit > 0 else ""
        rows = db.execute(text(f"""
            SELECT r.id, r.candidate_control_id, r.candidate_title,
                   r.matched_control_id, r.similarity_score,
                   c1.objective as candidate_objective,
                   c1.requirements::text as candidate_requirements,
                   c2.title as matched_title,
                   c2.objective as matched_objective,
                   c2.requirements::text as matched_requirements
            FROM compliance.control_dedup_reviews r
            LEFT JOIN compliance.canonical_controls c1 ON c1.control_id = r.candidate_control_id
            LEFT JOIN compliance.canonical_controls c2 ON c2.control_id = r.matched_control_id
            WHERE r.dedup_stage = 'batch_dedup'
              AND r.similarity_score >= :min_score
              AND r.similarity_score < :max_score
            ORDER BY r.similarity_score DESC
            {limit_clause}
        """), {"min_score": req.min_score, "max_score": req.max_score}).fetchall()

        total = len(rows)
        duplicates = 0
        different = 0
        errors = 0
        results = []

        _llm_dedup_status[job_id] = {
            "status": "running", "total": total, "processed": 0,
            "duplicates": 0, "different": 0, "errors": 0,
            "dry_run": req.dry_run,
        }

        for i, row in enumerate(rows):
            try:
                # Build comparison prompt
                candidate_ctx = row.candidate_title or ""
                if row.candidate_objective:
                    candidate_ctx += f"\nObjective: {row.candidate_objective[:300]}"
                if row.candidate_requirements and row.candidate_requirements not in ("[]", "null"):
                    candidate_ctx += f"\nRequirements: {row.candidate_requirements[:300]}"

                matched_ctx = row.matched_title or ""
                if row.matched_objective:
                    matched_ctx += f"\nObjective: {row.matched_objective[:300]}"
                if row.matched_requirements and row.matched_requirements not in ("[]", "null"):
                    matched_ctx += f"\nRequirements: {row.matched_requirements[:300]}"

                prompt = f"Control A ({row.candidate_control_id}):\n{candidate_ctx}\n\nControl B ({row.matched_control_id}):\n{matched_ctx}\n\nSind diese Controls Duplikate?"

                async with httpx.AsyncClient(timeout=30.0) as client:
                    resp = await client.post(
                        f"{OLLAMA_URL}/api/chat",
                        json={
                            "model": req.model,
                            "stream": False,
                            "think": False,
                            "options": {"num_predict": 200},
                            "messages": [
                                {"role": "system", "content": "Du bist ein Compliance-Experte. Vergleiche zwei Controls und entscheide: DUPLIKAT (gleiche Anforderung, nur anders formuliert) oder VERSCHIEDEN (unterschiedlicher Scope/Inhalt). Antworte NUR mit einem JSON: {\"verdict\": \"DUPLIKAT\" oder \"VERSCHIEDEN\", \"reason\": \"kurze Begruendung\"}"},
                                {"role": "user", "content": prompt},
                            ],
                        },
                    )

                if resp.status_code != 200:
                    errors += 1
                    continue

                content = resp.json().get("message", {}).get("content", "")
                # Parse verdict from response
                parsed = _parse_llm_json(content)
                if not parsed:
                    errors += 1
                    continue

                verdict = parsed.get("verdict", "").upper()
                reason = parsed.get("reason", "")

                if "DUPLIKAT" in verdict:
                    duplicates += 1
                    if not req.dry_run:
                        # Mark as duplicate
                        db.execute(text("""
                            UPDATE compliance.canonical_controls
                            SET release_state = 'duplicate',
                                merged_into_uuid = (SELECT id FROM compliance.canonical_controls WHERE control_id = :matched LIMIT 1),
                                updated_at = NOW()
                            WHERE control_id = :candidate AND release_state = 'draft'
                        """), {"candidate": row.candidate_control_id, "matched": row.matched_control_id})
                else:
                    different += 1

                results.append({
                    "candidate": row.candidate_control_id,
                    "matched": row.matched_control_id,
                    "score": float(row.similarity_score),
                    "verdict": verdict,
                    "reason": reason[:100],
                })

            except Exception as e:
                errors += 1
                logger.warning("LLM dedup error for %s: %s", row.candidate_control_id, e)

            if not req.dry_run and (i + 1) % 50 == 0:
                db.commit()

            _llm_dedup_status[job_id] = {
                "status": "running", "total": total, "processed": i + 1,
                "duplicates": duplicates, "different": different, "errors": errors,
                "dry_run": req.dry_run,
            }

        if not req.dry_run:
            db.commit()

        _llm_dedup_status[job_id] = {
            "status": "completed", "total": total, "processed": total,
            "duplicates": duplicates, "different": different, "errors": errors,
            "dry_run": req.dry_run, "results": results[-50:],
        }
        logger.info("LLM dedup %s completed: %d dup, %d diff, %d err out of %d",
                     job_id, duplicates, different, errors, total)

    except Exception as e:
        logger.error("LLM dedup %s failed: %s", job_id, e)
        _llm_dedup_status[job_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/llm-dedup")
async def start_llm_dedup(req: LLMDedupRequest):
    """Use local LLM to verify borderline dedup matches from the review queue.

    Sends each candidate+matched pair to the local Ollama LLM for a
    DUPLIKAT/VERSCHIEDEN verdict. Much more accurate than embedding similarity.
    Default is dry_run=True (preview only, no DB changes).
    """
    import uuid
    job_id = str(uuid.uuid4())[:8]
    _llm_dedup_status[job_id] = {"status": "starting"}
    asyncio.create_task(_run_llm_dedup(req, job_id))
    return {
        "status": "running",
        "job_id": job_id,
        "message": f"LLM dedup started. Poll /generate/llm-dedup-status/{job_id}",
    }


@router.get("/generate/llm-dedup-status/{job_id}")
async def get_llm_dedup_status(job_id: str):
    """Get status of an LLM dedup job."""
    status = _llm_dedup_status.get(job_id)
    if not status:
        raise HTTPException(status_code=404, detail="LLM dedup job not found")
    return status


# =============================================================================
# ANCHOR BACKFILL
# =============================================================================

class AnchorBackfillRequest(BaseModel):
    dry_run: bool = True
    limit: int = 0  # 0 = all controls without anchors
    batch_size: int = 50
    skip_web: bool = True  # Stage A only (RAG), no DuckDuckGo
    include_needs_review: bool = True  # Also backfill needs_review controls


_anchor_backfill_status: dict = {}


async def _run_anchor_backfill(req: AnchorBackfillRequest, backfill_id: str):
    """Backfill open_anchors for controls that were generated with skip_web_search=true."""
    from dataclasses import asdict

    db = SessionLocal()
    try:
        finder = AnchorFinder()

        # Find controls without anchors
        states = "('draft', 'needs_review')" if req.include_needs_review else "('draft',)"
        limit_clause = f"LIMIT {req.limit}" if req.limit > 0 else ""
        rows = db.execute(text(f"""
            SELECT id, control_id, title, tags
            FROM compliance.canonical_controls
            WHERE release_state IN {states}
              AND (open_anchors IS NULL OR open_anchors::text = '[]'
                   OR open_anchors::text = 'null' OR open_anchors::text = '')
            ORDER BY control_id
            {limit_clause}
        """)).fetchall()

        total = len(rows)
        updated = 0
        with_anchors = 0
        empty_anchors = 0
        errors = []

        _anchor_backfill_status[backfill_id] = {
            "status": "running", "total": total, "updated": 0,
            "with_anchors": 0, "empty_anchors": 0, "dry_run": req.dry_run,
        }

        for i in range(0, total, req.batch_size):
            batch = rows[i:i + req.batch_size]

            for row in batch:
                try:
                    # Parse tags from DB (may be JSON string or list)
                    tags = row.tags
                    if isinstance(tags, str):
                        try:
                            tags = json.loads(tags)
                        except (json.JSONDecodeError, ValueError):
                            tags = []
                    if not isinstance(tags, list):
                        tags = []

                    # Build minimal GeneratedControl for AnchorFinder
                    control = _GeneratedControl(
                        title=row.title or "",
                        tags=tags,
                    )

                    # Find anchors (Stage A: RAG only)
                    anchors = await finder.find_anchors(
                        control, skip_web=req.skip_web, min_anchors=2
                    )

                    anchor_dicts = [asdict(a) for a in anchors]

                    if not req.dry_run:
                        db.execute(text("""
                            UPDATE compliance.canonical_controls
                            SET open_anchors = CAST(:anchors AS jsonb),
                                updated_at = NOW()
                            WHERE id = CAST(:cid AS uuid)
                        """), {"anchors": json.dumps(anchor_dicts), "cid": str(row.id)})

                    updated += 1
                    if anchor_dicts:
                        with_anchors += 1
                    else:
                        empty_anchors += 1

                except Exception as e:
                    errors.append(f"{row.control_id}: {str(e)[:200]}")
                    logger.warning("Anchor backfill error for %s: %s", row.control_id, e)

            if not req.dry_run:
                db.commit()

            _anchor_backfill_status[backfill_id] = {
                "status": "running", "total": total, "updated": updated,
                "with_anchors": with_anchors, "empty_anchors": empty_anchors,
                "progress": f"{min(i + req.batch_size, total)}/{total}",
                "dry_run": req.dry_run, "errors": errors[-10:],
            }

        # Promote needs_review controls that gained anchors to draft
        promoted = 0
        if not req.dry_run and req.include_needs_review:
            result = db.execute(text("""
                UPDATE compliance.canonical_controls
                SET release_state = 'draft', updated_at = NOW()
                WHERE release_state = 'needs_review'
                  AND open_anchors IS NOT NULL
                  AND open_anchors::text != '[]'
                  AND open_anchors::text != 'null'
                  AND open_anchors::text != ''
                RETURNING id
            """))
            promoted = result.rowcount
            db.commit()

        _anchor_backfill_status[backfill_id] = {
            "status": "completed", "total": total, "updated": updated,
            "with_anchors": with_anchors, "empty_anchors": empty_anchors,
            "promoted_to_draft": promoted,
            "dry_run": req.dry_run, "errors": errors[-50:],
        }
        logger.info("Anchor backfill %s completed: %d/%d updated (%d with anchors, %d empty, %d promoted)",
                     backfill_id, updated, total, with_anchors, empty_anchors, promoted)

    except Exception as e:
        logger.error("Anchor backfill %s failed: %s", backfill_id, e)
        _anchor_backfill_status[backfill_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/backfill-anchors")
async def start_anchor_backfill(req: AnchorBackfillRequest):
    """Backfill open_anchors (OWASP/NIST/ENISA references) for controls without anchors.

    Uses RAG-internal search (Stage A) to find open-source framework references.
    Controls generated with skip_web_search=true have empty open_anchors.
    Default is dry_run=True (preview only, no DB changes).
    """
    import uuid
    backfill_id = str(uuid.uuid4())[:8]
    _anchor_backfill_status[backfill_id] = {"status": "starting"}
    asyncio.create_task(_run_anchor_backfill(req, backfill_id))
    return {
        "status": "running",
        "backfill_id": backfill_id,
        "message": f"Anchor backfill started. Poll /generate/anchor-backfill-status/{backfill_id}",
    }


@router.get("/generate/anchor-backfill-status/{backfill_id}")
async def get_anchor_backfill_status(backfill_id: str):
    """Get status of an anchor backfill job."""
    status = _anchor_backfill_status.get(backfill_id)
    if not status:
        raise HTTPException(status_code=404, detail="Anchor backfill job not found")
    return status


# =============================================================================
# HARMONIZATION RECHECK — index ALL drafts, then check target controls
# =============================================================================

class HarmonizationRecheckRequest(BaseModel):
    dry_run: bool = True
    since: str = "2026-04-24 08:30:00"
    until: str = "2026-04-24 09:00:00"


_harmonization_recheck_status: dict = {}

RECHECK_COLLECTION = "draft_controls_recheck"


async def _run_harmonization_recheck(req: HarmonizationRecheckRequest, job_id: str):
    """Two-phase recheck: (1) index ALL drafts, (2) search target controls against them."""
    import os
    import httpx
    import uuid as _uuid

    QDRANT_URL = os.getenv("QDRANT_URL", "http://qdrant:6333")
    EMBEDDING_URL = os.getenv("EMBEDDING_URL", "http://embedding-service:8087")
    OLLAMA_URL = os.getenv("OLLAMA_URL", "http://host.docker.internal:11434")
    OLLAMA_MODEL = os.getenv("CONTROL_GEN_OLLAMA_MODEL", "qwen3.5:35b-a3b")
    AUTO_DUP = 0.92
    THRESHOLD = 0.85

    db = SessionLocal()
    try:
        # Load target controls (the ones we want to check)
        targets = db.execute(text("""
            SELECT id::text, control_id, title, objective
            FROM compliance.canonical_controls
            WHERE release_state = 'draft'
              AND updated_at >= CAST(:since AS timestamp)
              AND updated_at < CAST(:until AS timestamp)
            ORDER BY control_id
        """), {"since": req.since, "until": req.until}).fetchall()

        target_ids = {r[0] for r in targets}
        total_targets = len(targets)

        # Count reference controls (all drafts except targets)
        total_reference = db.execute(text("""
            SELECT COUNT(*) FROM compliance.canonical_controls
            WHERE release_state = 'draft'
              AND NOT (updated_at >= CAST(:since AS timestamp) AND updated_at < CAST(:until AS timestamp))
        """), {"since": req.since, "until": req.until}).scalar()

        _harmonization_recheck_status[job_id] = {
            "status": "phase1_indexing", "total_targets": total_targets,
            "total_reference": total_reference, "indexed": 0,
            "processed": 0, "unique": 0, "duplicate": 0,
            "llm_calls": 0, "errors": 0, "dry_run": req.dry_run,
        }

        logger.info("Harmonization recheck: %d targets, %d reference controls", total_targets, total_reference)

        # Phase 1: Create temporary Qdrant collection and index reference controls
        async with httpx.AsyncClient(timeout=30.0) as client:
            # Delete old collection if exists
            await client.delete(f"{QDRANT_URL}/collections/{RECHECK_COLLECTION}")

            # Get embedding dimension
            resp = await client.post(f"{EMBEDDING_URL}/embed", json={"texts": ["test"]})
            dim = len(resp.json().get("embeddings", [[]])[0])

            # Create collection
            await client.put(f"{QDRANT_URL}/collections/{RECHECK_COLLECTION}", json={
                "vectors": {"size": dim, "distance": "Cosine"},
            })

        # Index reference controls in paginated batches (never load all into memory)
        EMBED_BATCH = 8
        DB_PAGE = 500
        indexed = 0
        db_offset = 0

        while db_offset < total_reference + DB_PAGE:
            page_rows = db.execute(text("""
                SELECT id::text, control_id, title, objective
                FROM compliance.canonical_controls
                WHERE release_state = 'draft'
                  AND NOT (updated_at >= CAST(:since AS timestamp) AND updated_at < CAST(:until AS timestamp))
                ORDER BY control_id
                LIMIT :lim OFFSET :off
            """), {"since": req.since, "until": req.until, "lim": DB_PAGE, "off": db_offset}).fetchall()

            if not page_rows:
                break

            for eb_start in range(0, len(page_rows), EMBED_BATCH):
                batch = page_rows[eb_start:eb_start + EMBED_BATCH]
                texts = [f"{r[2] or ''} {(r[3] or '')[:200]}" for r in batch]

                try:
                    async with httpx.AsyncClient(timeout=30.0) as client:
                        resp = await client.post(f"{EMBEDDING_URL}/embed", json={"texts": texts})
                        if resp.status_code != 200:
                            continue
                        embeddings = resp.json().get("embeddings", [])

                        points = []
                        for r, emb in zip(batch, embeddings):
                            if not emb:
                                continue
                            points.append({
                                "id": str(_uuid.uuid5(_uuid.NAMESPACE_DNS, r[0])),
                                "vector": emb,
                                "payload": {"control_uuid": r[0], "control_id": r[1], "title": r[2] or ""},
                            })

                        if points:
                            await client.put(
                                f"{QDRANT_URL}/collections/{RECHECK_COLLECTION}/points",
                                json={"points": points},
                            )
                            indexed += len(points)
                except Exception as e:
                    logger.warning("Indexing batch error at offset %d: %s", db_offset + eb_start, e)

            db_offset += DB_PAGE
            _harmonization_recheck_status[job_id]["indexed"] = indexed
            _harmonization_recheck_status[job_id]["status"] = "phase1_indexing"
            if db_offset % 5000 < DB_PAGE:
                logger.info("Recheck indexing: %d/%d", indexed, total_reference)

        logger.info("Recheck Phase 1 done: %d reference controls indexed", indexed)

        # Phase 2: Check each target against the reference collection
        _harmonization_recheck_status[job_id]["status"] = "phase2_checking"
        unique = 0
        duplicate = 0
        llm_calls = 0
        no_match = 0
        errors = 0

        for i, row in enumerate(targets):
            try:
                search_text = f"{row[2] or ''} {(row[3] or '')[:200]}"

                async with httpx.AsyncClient(timeout=15.0) as client:
                    resp = await client.post(f"{EMBEDDING_URL}/embed",
                                             json={"texts": [search_text]})
                    if resp.status_code != 200:
                        errors += 1
                        continue
                    emb = resp.json().get("embeddings", [[]])[0]
                    if not emb:
                        errors += 1
                        continue

                async with httpx.AsyncClient(timeout=15.0) as client:
                    resp = await client.post(
                        f"{QDRANT_URL}/collections/{RECHECK_COLLECTION}/points/search",
                        json={"vector": emb, "limit": 3,
                              "score_threshold": THRESHOLD,
                              "with_payload": {"include": ["control_id", "title"]}})
                    results = resp.json().get("result", []) if resp.status_code == 200 else []

                if not results:
                    no_match += 1
                    unique += 1
                    continue

                best_score = results[0].get("score", 0.0)
                best_title = results[0].get("payload", {}).get("title", "")

                if best_score >= AUTO_DUP:
                    if not req.dry_run:
                        db.execute(text(
                            "UPDATE compliance.canonical_controls "
                            "SET release_state = 'duplicate', updated_at = NOW() "
                            "WHERE id = CAST(:id AS uuid)"
                        ), {"id": row[0]})
                    duplicate += 1

                elif best_score >= THRESHOLD:
                    try:
                        async with httpx.AsyncClient(timeout=30.0) as client:
                            resp = await client.post(f"{OLLAMA_URL}/api/chat", json={
                                "model": OLLAMA_MODEL, "stream": False, "think": False,
                                "options": {"num_predict": 200},
                                "messages": [
                                    {"role": "system", "content": (
                                        "Vergleiche zwei Controls: DUPLIKAT oder VERSCHIEDEN. "
                                        'Antworte NUR mit JSON: {"verdict":"DUPLIKAT" oder "VERSCHIEDEN","reason":"..."}'
                                    )},
                                    {"role": "user", "content": (
                                        f"Control A:\n{row[2] or ''}\n\n"
                                        f"Control B:\n{best_title}\n\nDuplikat?"
                                    )},
                                ],
                            })
                        llm_calls += 1
                        content = resp.json().get("message", {}).get("content", "")
                        parsed = _parse_llm_json(content)
                        if parsed and "DUPLIKAT" in str(parsed.get("verdict", "")).upper():
                            if not req.dry_run:
                                db.execute(text(
                                    "UPDATE compliance.canonical_controls "
                                    "SET release_state = 'duplicate', updated_at = NOW() "
                                    "WHERE id = CAST(:id AS uuid)"
                                ), {"id": row[0]})
                            duplicate += 1
                        else:
                            unique += 1
                    except Exception:
                        unique += 1
                        errors += 1
                else:
                    unique += 1

            except Exception as e:
                errors += 1
                logger.warning("Recheck error %s: %s", row[1], e)

            if (i + 1) % 50 == 0:
                if not req.dry_run:
                    db.commit()
                _harmonization_recheck_status[job_id] = {
                    "status": "phase2_checking", "total_targets": total_targets,
                    "total_reference": total_reference, "indexed": indexed,
                    "processed": i + 1, "unique": unique, "duplicate": duplicate,
                    "llm_calls": llm_calls, "no_match": no_match,
                    "errors": errors, "dry_run": req.dry_run,
                }

        if not req.dry_run:
            db.commit()

        # Cleanup temporary collection
        async with httpx.AsyncClient(timeout=30.0) as client:
            await client.delete(f"{QDRANT_URL}/collections/{RECHECK_COLLECTION}")

        _harmonization_recheck_status[job_id] = {
            "status": "completed", "total_targets": total_targets,
            "total_reference": total_reference, "indexed": indexed,
            "processed": total_targets, "unique": unique, "duplicate": duplicate,
            "llm_calls": llm_calls, "no_match": no_match,
            "errors": errors, "dry_run": req.dry_run,
        }
        logger.info("Recheck DONE: %d targets, %d unique, %d dup, %d llm, %d err",
                     total_targets, unique, duplicate, llm_calls, errors)

    except Exception as e:
        logger.error("Harmonization recheck %s failed: %s", job_id, e)
        _harmonization_recheck_status[job_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/harmonization-recheck")
async def start_harmonization_recheck(req: HarmonizationRecheckRequest):
    """Re-check promoted controls against ALL other draft controls.
    Phase 1: Index all non-target drafts into temp Qdrant collection.
    Phase 2: Search each target control, Embedding + LLM for borderline.
    """
    import uuid
    job_id = str(uuid.uuid4())[:8]
    _harmonization_recheck_status[job_id] = {"status": "starting"}
    asyncio.create_task(_run_harmonization_recheck(req, job_id))
    return {
        "status": "running", "job_id": job_id,
        "message": f"Harmonization recheck started. Poll /generate/harmonization-recheck-status/{job_id}",
    }


@router.get("/generate/harmonization-recheck-status/{job_id}")
async def get_harmonization_recheck_status(job_id: str):
    status = _harmonization_recheck_status.get(job_id)
    if not status:
        raise HTTPException(status_code=404, detail="Harmonization recheck job not found")
    return status


# =============================================================================
# BATCH API: STATUS + RESULT PROCESSING
# =============================================================================

@router.get("/generate/batch-api-status/{batch_id}")
async def get_batch_api_status(batch_id: str):
    """Check status of an Anthropic Batch API job."""
    from services.decomposition_pass import check_batch_status
    status = await check_batch_status(batch_id)
    return status


class ProcessBatchRequest(BaseModel):
    batch_id: str
    pass_type: str = "0b"  # "0a" or "0b"


_batch_process_status: dict = {}


async def _run_batch_processing(req: ProcessBatchRequest, job_id: str):
    """Process batch results in background."""
    from services.decomposition_pass import DecompositionPass
    db = SessionLocal()
    try:
        _batch_process_status[job_id] = {"status": "running", "phase": "processing"}
        dp = DecompositionPass(db)
        result = await dp.process_batch_results(req.batch_id, req.pass_type)
        _batch_process_status[job_id] = {"status": "completed", **result}
    except Exception as e:
        logger.error("Batch processing %s failed: %s", job_id, e)
        _batch_process_status[job_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/process-batch")
async def process_batch(req: ProcessBatchRequest):
    """Process results from a completed Anthropic Batch API job."""
    import uuid
    job_id = str(uuid.uuid4())[:8]
    _batch_process_status[job_id] = {"status": "starting"}
    asyncio.create_task(_run_batch_processing(req, job_id))
    return {
        "status": "running", "job_id": job_id,
        "message": f"Batch processing started. Poll /generate/process-batch-status/{job_id}",
    }


@router.get("/generate/process-batch-status/{job_id}")
async def get_batch_process_status(job_id: str):
    """Get status of batch result processing."""
    status = _batch_process_status.get(job_id)
    if not status:
        raise HTTPException(status_code=404, detail="Batch processing job not found")
    return status


class RunPass0aRequest(BaseModel):
    limit: int = 0  # 0 = no limit
    batch_size: int = 5
    use_anthropic: bool = True
    category_filter: Optional[str] = None
    source_filter: Optional[str] = None


_pass0a_status: dict = {}


async def _run_pass0a_background(req: RunPass0aRequest, job_id: str):
    """Run Pass 0a in background with own DB session."""
    from services.decomposition_pass import DecompositionPass
    db = SessionLocal()
    try:
        _pass0a_status[job_id] = {"status": "running"}
        dp = DecompositionPass(db)
        result = await dp.run_pass0a(
            limit=req.limit,
            batch_size=req.batch_size,
            use_anthropic=req.use_anthropic,
            category_filter=req.category_filter,
            source_filter=req.source_filter,
        )
        _pass0a_status[job_id] = {"status": "completed", **result}
        logger.info("Pass 0a job %s completed: %s", job_id, result)
    except Exception as e:
        logger.error("Pass 0a job %s failed: %s", job_id, e)
        _pass0a_status[job_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/run-pass0a")
async def run_pass0a(req: RunPass0aRequest):
    """Run Pass 0a (Obligation Extraction) on undecomposed controls.

    Extracts individual normative obligations from rich controls using LLM.
    Runs in background — poll status via GET /generate/pass0a-status/{job_id}.
    """
    import uuid
    job_id = str(uuid.uuid4())[:8]
    _pass0a_status[job_id] = {"status": "starting"}
    asyncio.create_task(_run_pass0a_background(req, job_id))
    return {
        "status": "running",
        "job_id": job_id,
        "message": f"Pass 0a started. Poll /generate/pass0a-status/{job_id}",
    }


@router.get("/generate/pass0a-status/{job_id}")
async def get_pass0a_status(job_id: str):
    """Get status of a Pass 0a job."""
    status = _pass0a_status.get(job_id)
    if not status:
        raise HTTPException(status_code=404, detail="Pass 0a job not found")
    return status


class SubmitPass0bRequest(BaseModel):
    limit: int = 10
    batch_size: int = 5


_last_submit_batch_id: str = ""
_last_submit_time: float = 0


@router.post("/generate/submit-pass0b")
async def submit_pass0b(req: SubmitPass0bRequest):
    """Submit Pass 0b batch to Anthropic Batch API.

    SAFETY: Refuses to submit if a batch was submitted in the last 10 minutes.
    This prevents duplicate batches from curl retries or timeouts.
    """
    import time
    global _last_submit_batch_id, _last_submit_time

    # Idempotency guard: refuse if last submit was <10 min ago
    elapsed = time.time() - _last_submit_time
    if elapsed < 600 and _last_submit_batch_id:
        return {
            "status": "blocked",
            "reason": f"Batch {_last_submit_batch_id} was submitted {int(elapsed)}s ago. Wait {int(600 - elapsed)}s or use force=true.",
            "last_batch_id": _last_submit_batch_id,
        }

    from services.decomposition_pass import DecompositionPass
    db = SessionLocal()
    try:
        dp = DecompositionPass(db)
        result = await dp.submit_batch_pass0b(
            limit=req.limit,
            batch_size=req.batch_size,
        )
        # Record successful submit
        batch_id = result.get("batch_id", "")
        if batch_id:
            _last_submit_batch_id = batch_id
            _last_submit_time = time.time()
            logger.info("Submit guard: recorded batch %s", batch_id)
        return result
    except Exception as e:
        logger.error("Submit Pass 0b failed: %s", e)
        raise HTTPException(status_code=500, detail=str(e))
    finally:
        db.close()


# =============================================================================
# HAIKU BACKFILL: Extended Fields (applicability, check_type, scanner_hint, etc.)
# =============================================================================

_BACKFILL_SYSTEM_PROMPT = """\
Du bist ein Security-Compliance-Experte. Du erhaeltst bestehende atomare Controls \
und ergaenzt fehlende Felder. Aendere NIEMALS bestehende Felder (title, assertion, \
objective, requirements, evidence, severity, category).

Ergaenze NUR diese 6 Felder:

1. applicability: Unter welchen Bedingungen gilt dieses Control?
   Universell: {}
   Bedingt: {"field": "context.SIGNAL", "op": "==", "value": true}
   Zusammengesetzt: {"operator": "AND", "clauses": [{...}, {...}]}
   Typische Felder: context.uses_oauth, context.has_public_api,
   context.processes_personal_data, context.uses_ai_system,
   context.has_employees, context.sells_online, context.uses_encryption

2. check_type: EINEN der 10 Werte:
   technical_config_check, code_pattern_check, runtime_security_test,
   document_policy_check, document_classification_check, document_contract_check,
   evidence_artifact_check, process_verification, training_verification, interview_assessment

3. scanner_hint: {"search_terms": [...], "negative_indicators": [...]}

4. manual_review_required_if: ["Bedingung 1", "Bedingung 2"]

5. evidence_type: code|process|hybrid

6. provides_context: ["context.VARIABLE", ...] oder []

Antworte als JSON-Objekt mit Control-ID als Key."""


class BackfillExtendedRequest(BaseModel):
    limit: int = 0
    batch_size: int = 10
    dry_run: bool = True


_backfill_extended_status: dict = {}


async def _run_backfill_extended(req: BackfillExtendedRequest, job_id: str):
    """Run extended field backfill via Haiku Batch API."""
    from services.decomposition_pass import (
        create_anthropic_batch, fetch_batch_results, check_batch_status,
    )
    db = SessionLocal()
    try:
        _backfill_extended_status[job_id] = {"status": "loading_controls"}

        query = """
            SELECT id::text, control_id, title,
                   generation_metadata->>'assertion' as assertion,
                   objective, category, severity,
                   generation_metadata->>'merge_group_hint' as merge_key
            FROM canonical_controls
            WHERE release_state = 'draft'
              AND generation_metadata->>'decomposition_method' = 'pass0b'
              AND (generation_metadata->>'applicability' IS NULL
                   OR generation_metadata->>'applicability' = ''
                   OR generation_metadata->>'applicability' = '{}')
        """
        if req.limit > 0:
            query += f" LIMIT {req.limit}"

        rows = db.execute(text(query)).fetchall()
        total = len(rows)
        _backfill_extended_status[job_id] = {
            "status": "preparing_batches", "total_controls": total,
        }

        if total == 0:
            _backfill_extended_status[job_id] = {
                "status": "completed", "total_controls": 0, "message": "Nothing to backfill",
            }
            return

        if req.dry_run:
            _backfill_extended_status[job_id] = {
                "status": "dry_run_complete", "total_controls": total,
                "estimated_requests": (total + req.batch_size - 1) // req.batch_size,
            }
            return

        # Build batch requests
        requests = []
        for i in range(0, total, req.batch_size):
            batch = rows[i:i + req.batch_size]
            controls_text = ""
            for r in batch:
                controls_text += f"""
Control-ID: {r[1]}
Titel: {r[2]}
Assertion: {r[3] or ''}
Objective: {r[4] or ''}
Kategorie: {r[5] or ''}
Severity: {r[6] or ''}
Merge-Key: {r[7] or ''}
---
"""
            prompt = f"Ergaenze die fehlenden 6 Felder fuer diese {len(batch)} Controls:\n{controls_text}"

            batch_idx = i // req.batch_size
            requests.append({
                "custom_id": f"bf_ext_b{batch_idx:05d}",
                "params": {
                    "model": "claude-haiku-4-5-20251001",
                    "max_tokens": max(2048, len(batch) * 400),
                    "system": [
                        {
                            "type": "text",
                            "text": _BACKFILL_SYSTEM_PROMPT,
                            "cache_control": {"type": "ephemeral"},
                        }
                    ],
                    "messages": [{"role": "user", "content": prompt}],
                },
            })

        _backfill_extended_status[job_id] = {
            "status": "submitting_batch", "total_controls": total,
            "total_requests": len(requests),
        }

        batch_result = await create_anthropic_batch(requests)
        batch_id = batch_result.get("id", "")

        _backfill_extended_status[job_id] = {
            "status": "batch_submitted", "batch_id": batch_id,
            "total_controls": total, "total_requests": len(requests),
        }

        # Poll for completion
        import asyncio
        for _ in range(360):  # max 1 hour
            await asyncio.sleep(10)
            status = await check_batch_status(batch_id)
            if status.get("processing_status") == "ended":
                break

        # Process results
        results = await fetch_batch_results(batch_id)
        updated = 0
        errors = 0

        for result in results:
            result_data = result.get("result", {})
            if result_data.get("type") != "succeeded":
                errors += 1
                continue

            message = result_data.get("message", {})
            content = message.get("content", [])
            text_content = content[0].get("text", "") if content else ""

            try:
                import json as json_mod
                # Try to parse JSON object
                parsed = json_mod.loads(text_content) if text_content.strip().startswith("{") else {}
                if not parsed:
                    # Try extracting JSON from markdown
                    import re
                    json_match = re.search(r'\{[\s\S]*\}', text_content)
                    if json_match:
                        parsed = json_mod.loads(json_match.group())

                for control_id, fields in parsed.items():
                    if not isinstance(fields, dict):
                        continue
                    # Merge into generation_metadata
                    db.execute(text("""
                        UPDATE canonical_controls
                        SET generation_metadata = generation_metadata || CAST(:new_fields AS jsonb),
                            updated_at = NOW()
                        WHERE control_id = :cid
                          AND release_state = 'draft'
                    """), {
                        "cid": control_id,
                        "new_fields": json_mod.dumps({
                            "applicability": fields.get("applicability", {}),
                            "check_type": fields.get("check_type", ""),
                            "scanner_hint": fields.get("scanner_hint", {}),
                            "manual_review_required_if": fields.get("manual_review_required_if", []),
                            "evidence_type": fields.get("evidence_type", ""),
                            "provides_context": fields.get("provides_context", []),
                        }),
                    })
                    updated += 1
                db.commit()
            except Exception as e:
                logger.error("Backfill parse error: %s", e)
                errors += 1
                try:
                    db.rollback()
                except Exception:
                    pass

        _backfill_extended_status[job_id] = {
            "status": "completed", "batch_id": batch_id,
            "total_controls": total, "updated": updated, "errors": errors,
        }
    except Exception as e:
        logger.error("Backfill extended %s failed: %s", job_id, e)
        _backfill_extended_status[job_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/backfill-extended")
async def start_backfill_extended(req: BackfillExtendedRequest):
    """Backfill extended fields (applicability, scanner_hint, etc.) via Haiku Batch API."""
    import uuid as uuid_mod
    job_id = str(uuid_mod.uuid4())[:8]
    _backfill_extended_status[job_id] = {"status": "starting"}
    asyncio.create_task(_run_backfill_extended(req, job_id))
    return {
        "status": "running", "job_id": job_id,
        "message": f"Backfill started. Poll /generate/backfill-extended-status/{job_id}",
    }


@router.get("/generate/backfill-extended-status/{job_id}")
async def get_backfill_extended_status(job_id: str):
    status = _backfill_extended_status.get(job_id)
    if not status:
        raise HTTPException(status_code=404, detail="Backfill job not found")
    return status


@router.get("/generate/quality-metrics")
async def get_quality_metrics(
    since_hours: int = Query(default=0, description="Only count controls created in last N hours (0=all)"),
):
    """Quality metrics for Pass 0b atomic controls.

    Measures:
    - controls_per_obligation: ratio of controls to obligations (target: 0.2-0.4)
    - duplicate_rate: controls sharing merge_key (target: <5%)
    - evidence_leak_rate: controls with evidence-like titles (target: <2%)
    - container_rate: controls with composite-like titles (target: <10%)
    - truncated_title_rate: titles ending abruptly (target: 0%)
    - mcp_coverage: controls with assertion + pass/fail criteria
    """
    db = SessionLocal()
    try:
        time_filter = ""
        if since_hours > 0:
            time_filter = f"AND cc.created_at > NOW() - interval '{since_hours} hours'"

        # Total pass0b draft controls
        total = db.execute(text(f"""
            SELECT COUNT(*) FROM compliance.canonical_controls cc
            WHERE cc.release_state = 'draft'
              AND cc.generation_metadata->>'decomposition_method' = 'pass0b'
              {time_filter}
        """)).scalar() or 0

        # Total validated obligations
        total_obligations = db.execute(text("""
            SELECT COUNT(*) FROM compliance.obligation_candidates
            WHERE release_state = 'validated'
        """)).scalar() or 0

        if total == 0:
            return {
                "total_controls": 0,
                "total_obligations": total_obligations,
                "message": "No pass0b controls found",
            }

        # Controls per obligation
        controls_per_obligation = round(total / max(total_obligations, 1), 3)

        # Duplicate rate: merge_keys that appear >1 time
        dup_rows = db.execute(text(f"""
            SELECT COUNT(*) as dup_count FROM (
                SELECT generation_metadata->>'merge_group_hint' as mk, COUNT(*) as cnt
                FROM compliance.canonical_controls cc
                WHERE cc.release_state = 'draft'
                  AND cc.generation_metadata->>'decomposition_method' = 'pass0b'
                  AND generation_metadata->>'merge_group_hint' IS NOT NULL
                  AND generation_metadata->>'merge_group_hint' != ''
                  {time_filter}
                GROUP BY mk
                HAVING COUNT(*) > 1
            ) sub
        """)).scalar() or 0

        total_with_mk = db.execute(text(f"""
            SELECT COUNT(DISTINCT generation_metadata->>'merge_group_hint')
            FROM compliance.canonical_controls cc
            WHERE cc.release_state = 'draft'
              AND cc.generation_metadata->>'decomposition_method' = 'pass0b'
              AND generation_metadata->>'merge_group_hint' IS NOT NULL
              AND generation_metadata->>'merge_group_hint' != ''
              {time_filter}
        """)).scalar() or 0

        duplicate_rate = round(dup_rows / max(total_with_mk, 1) * 100, 1)

        # Evidence leak rate: controls whose TITLE starts with evidence-type nouns
        # "nachweisen" (verb) is a valid action; "Nachweis vorliegen" is evidence
        evidence_start_patterns = (
            "nachweis vorli", "nachweis muss vorl", "screenshot",
            "export der", "auditbericht", "prüfbericht",
            "log-auszug", "jira-ticket", "sbom-nachweis",
        )
        evidence_count = 0
        rows = db.execute(text(f"""
            SELECT title FROM compliance.canonical_controls cc
            WHERE cc.release_state = 'draft'
              AND cc.generation_metadata->>'decomposition_method' = 'pass0b'
              {time_filter}
        """)).fetchall()
        for row in rows:
            title_lower = (row[0] or "").lower()
            if any(title_lower.startswith(p) for p in evidence_start_patterns):
                evidence_count += 1
        evidence_leak_rate = round(evidence_count / max(total, 1) * 100, 1)

        # Truncated title rate: titles cut off mid-word
        # True truncation: ends with incomplete word fragments (no space before last char)
        # NOT truncation: German titles ending with complete verbs (infinitives etc.)
        truncated_count = 0
        for row in rows:
            title = (row[0] or "").strip()
            if title and len(title) >= 195:
                # Near the 200-char limit — likely truncated
                truncated_count += 1
            elif title and len(title) >= 75:
                # Check for mid-word cutoff: last "word" has <3 chars and title is long
                last_word = title.split()[-1] if title.split() else ""
                if len(last_word) <= 2 and not last_word.endswith((".", "!", "?")):
                    truncated_count += 1
        truncated_title_rate = round(truncated_count / max(total, 1) * 100, 1)

        # MCP coverage
        mcp_count = db.execute(text(f"""
            SELECT COUNT(*) FROM compliance.canonical_controls cc
            WHERE cc.release_state = 'draft'
              AND cc.generation_metadata->>'decomposition_method' = 'pass0b'
              AND cc.generation_metadata->>'assertion' IS NOT NULL
              AND cc.generation_metadata->>'assertion' != ''
              {time_filter}
        """)).scalar() or 0
        mcp_coverage = round(mcp_count / max(total, 1) * 100, 1)

        # Merge key coverage
        mk_coverage = round(total_with_mk / max(total, 1) * 100, 1) if total_with_mk else 0

        return {
            "total_controls": total,
            "total_obligations": total_obligations,
            "controls_per_obligation": controls_per_obligation,
            "duplicate_merge_key_rate": f"{duplicate_rate}%",
            "evidence_leak_rate": f"{evidence_leak_rate}%",
            "truncated_title_rate": f"{truncated_title_rate}%",
            "mcp_coverage": f"{mcp_coverage}%",
            "merge_key_coverage": f"{mk_coverage}%",
            "targets": {
                "controls_per_obligation": "0.2-0.4",
                "duplicate_rate": "<5%",
                "evidence_leak_rate": "<2%",
                "truncated_title_rate": "0%",
                "mcp_coverage": "100%",
            },
        }
    finally:
        db.close()


# =============================================================================
# REVIEW CANDIDATE VERIFICATION (Block B — LLM decides DUPLIKAT/VERSCHIEDEN)
# =============================================================================

_REVIEW_VERIFY_SYSTEM = """Du vergleichst Paare von Compliance Controls und entscheidest ob sie Duplikate sind.
Antworte NUR mit einem JSON-Array. Fuer jedes Paar ein Objekt:
{"pair_id": "...", "decision": "DUPLIKAT" oder "VERSCHIEDEN", "reason": "kurze Begruendung"}
DUPLIKAT = gleiche Anforderung, nur anders formuliert.
VERSCHIEDEN = unterschiedliche Anforderungen, auch wenn aehnliche Woerter vorkommen."""


class ReviewVerifyRequest(BaseModel):
    limit: int = 0
    batch_size: int = 10
    dry_run: bool = True


_review_verify_status: dict = {}


async def _run_review_verify(req: ReviewVerifyRequest, job_id: str):
    from services.decomposition_pass import (
        create_anthropic_batch, fetch_batch_results, check_batch_status,
    )
    import asyncio as aio
    db = SessionLocal()
    try:
        _review_verify_status[job_id] = {"status": "loading"}

        query = """
            SELECT r.id::text, r.candidate_control_id, r.candidate_title,
                   r.matched_control_id, c2.title as matched_title,
                   r.similarity_score
            FROM control_dedup_reviews r
            LEFT JOIN canonical_controls c2 ON c2.id = r.matched_control_uuid
            WHERE r.review_status = 'pending'
            ORDER BY r.similarity_score DESC
        """
        if req.limit > 0:
            query += f" LIMIT {req.limit}"

        rows = db.execute(text(query)).fetchall()
        total = len(rows)
        _review_verify_status[job_id] = {"status": "preparing", "total": total}

        if total == 0:
            _review_verify_status[job_id] = {
                "status": "completed", "total": 0, "message": "No pending reviews",
            }
            return

        if req.dry_run:
            _review_verify_status[job_id] = {
                "status": "dry_run", "total": total,
                "estimated_requests": (total + req.batch_size - 1) // req.batch_size,
            }
            return

        # Build batch requests
        api_requests = []
        pair_map = {}
        for i in range(0, total, req.batch_size):
            batch = rows[i:i + req.batch_size]
            prompt = "Vergleiche diese Control-Paare:\n\n"
            batch_pairs = []
            for r in batch:
                pair_id = r[0][:8]
                prompt += (
                    f"Paar {pair_id}:\n"
                    f"  A: {r[1]} — {r[2]}\n"
                    f"  B: {r[3]} — {r[4]}\n"
                    f"  Similarity: {r[5]:.3f}\n\n"
                )
                batch_pairs.append({"review_id": r[0], "candidate_id": r[1]})

            batch_idx = i // req.batch_size
            custom_id = f"rv_b{batch_idx:05d}"
            pair_map[custom_id] = batch_pairs
            api_requests.append({
                "custom_id": custom_id,
                "params": {
                    "model": "claude-haiku-4-5-20251001",
                    "max_tokens": max(1024, len(batch) * 150),
                    "system": [{
                        "type": "text",
                        "text": _REVIEW_VERIFY_SYSTEM,
                        "cache_control": {"type": "ephemeral"},
                    }],
                    "messages": [{"role": "user", "content": prompt}],
                },
            })

        _review_verify_status[job_id] = {
            "status": "submitting", "total": total, "requests": len(api_requests),
        }
        batch_result = await create_anthropic_batch(api_requests)
        batch_id = batch_result.get("id", "")
        _review_verify_status[job_id] = {
            "status": "batch_submitted", "batch_id": batch_id,
            "total": total, "requests": len(api_requests),
        }

        # Poll for completion
        for _ in range(720):
            await aio.sleep(10)
            status = await check_batch_status(batch_id)
            if status.get("processing_status") == "ended":
                break

        # Process results
        results = await fetch_batch_results(batch_id)
        duplicates = 0
        different = 0
        errors = 0

        for result in results:
            custom_id = result.get("custom_id", "")
            result_data = result.get("result", {})
            if result_data.get("type") != "succeeded":
                errors += 1
                continue

            content = result_data.get("message", {}).get("content", [])
            text_content = content[0].get("text", "") if content else ""

            try:
                import json as jmod
                import re
                json_matches = re.findall(r'\{[^}]+\}', text_content)
                pairs = pair_map.get(custom_id, [])

                for j, match_str in enumerate(json_matches):
                    try:
                        parsed = jmod.loads(match_str)
                    except Exception:
                        continue

                    decision = parsed.get("decision", "").upper()
                    if j < len(pairs):
                        review_id = pairs[j]["review_id"]
                        if "DUPLIKAT" in decision:
                            db.execute(text("""
                                UPDATE control_dedup_reviews
                                SET review_status = 'duplicate', review_notes = :notes
                                WHERE id = CAST(:rid AS uuid)
                            """), {"rid": review_id, "notes": parsed.get("reason", "")})
                            duplicates += 1
                        else:
                            db.execute(text("""
                                UPDATE control_dedup_reviews
                                SET review_status = 'different', review_notes = :notes
                                WHERE id = CAST(:rid AS uuid)
                            """), {"rid": review_id, "notes": parsed.get("reason", "")})
                            different += 1

                db.commit()
            except Exception as e:
                logger.error("Review verify parse error: %s", e)
                errors += 1
                try:
                    db.rollback()
                except Exception:
                    pass

        _review_verify_status[job_id] = {
            "status": "completed", "batch_id": batch_id, "total": total,
            "duplicates": duplicates, "different": different, "errors": errors,
        }
    except Exception as e:
        logger.error("Review verify %s failed: %s", job_id, e)
        _review_verify_status[job_id] = {"status": "failed", "error": str(e)}
    finally:
        db.close()


@router.post("/generate/review-verify")
async def start_review_verify(req: ReviewVerifyRequest):
    """LLM-verify review candidates (DUPLIKAT/VERSCHIEDEN) via Haiku Batch."""
    import uuid as uuid_mod
    job_id = str(uuid_mod.uuid4())[:8]
    _review_verify_status[job_id] = {"status": "starting"}
    asyncio.create_task(_run_review_verify(req, job_id))
    return {
        "status": "running", "job_id": job_id,
        "message": f"Poll /generate/review-verify-status/{job_id}",
    }


@router.get("/generate/review-verify-status/{job_id}")
async def get_review_verify_status(job_id: str):
    status = _review_verify_status.get(job_id)
    if not status:
        raise HTTPException(status_code=404, detail="Review verify job not found")
    return status
