feat: add compliance modules 2-5 (dashboard, security templates, process manager, evidence collector)

Module 2: Extended Compliance Dashboard with roadmap, module-status, next-actions, snapshots, score-history
Module 3: 7 German security document templates (IT-Sicherheitskonzept, Datenschutz, Backup, Logging, Incident-Response, Zugriff, Risikomanagement)
Module 4: Compliance Process Manager with CRUD, complete/skip/seed, ~50 seed tasks, 3-tab UI
Module 5: Evidence Collector Extended with automated checks, control-mapping, coverage report, 4-tab UI

Also includes: canonical control library enhancements (verification method, categories, dedup), control generator improvements, RAG client extensions

52 tests pass, frontend builds clean.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

backend-compliance/compliance/api/__init__.py

@@ -53,6 +53,8 @@ _ROUTER_MODULES = [
     "wiki_routes",
     "canonical_control_routes",
     "control_generator_routes",
+    "process_task_routes",
+    "evidence_check_routes",
 ]
 
 _loaded_count = 0

backend-compliance/compliance/api/canonical_control_routes.py

@@ -78,6 +78,7 @@ class ControlResponse(BaseModel):
     customer_visible: Optional[bool] = None
     verification_method: Optional[str] = None
     category: Optional[str] = None
+    target_audience: Optional[str] = None
     generation_metadata: Optional[dict] = None
     created_at: str
     updated_at: str
@@ -106,6 +107,7 @@ class ControlCreateRequest(BaseModel):
     customer_visible: Optional[bool] = True
     verification_method: Optional[str] = None
     category: Optional[str] = None
+    target_audience: Optional[str] = None
     generation_metadata: Optional[dict] = None
 
 
@@ -130,6 +132,7 @@ class ControlUpdateRequest(BaseModel):
     customer_visible: Optional[bool] = None
     verification_method: Optional[str] = None
     category: Optional[str] = None
+    target_audience: Optional[str] = None
     generation_metadata: Optional[dict] = None
 
 
@@ -158,7 +161,7 @@ _CONTROL_COLS = """id, framework_id, control_id, title, objective, rationale,
                    evidence_confidence, open_anchors, release_state, tags,
                    license_rule, source_original_text, source_citation,
                    customer_visible, verification_method, category,
-                   generation_metadata,
+                   target_audience, generation_metadata,
                    created_at, updated_at"""
 
 
@@ -241,6 +244,7 @@ async def list_framework_controls(
     release_state: Optional[str] = Query(None),
     verification_method: Optional[str] = Query(None),
     category: Optional[str] = Query(None),
+    target_audience: Optional[str] = Query(None),
 ):
     """List controls belonging to a framework."""
     with SessionLocal() as db:
@@ -271,6 +275,9 @@ async def list_framework_controls(
         if category:
             query += " AND category = :cat"
             params["cat"] = category
+        if target_audience:
+            query += " AND target_audience = :ta"
+            params["ta"] = target_audience
 
         query += " ORDER BY control_id"
         rows = db.execute(text(query), params).fetchall()
@@ -289,6 +296,7 @@ async def list_controls(
     release_state: Optional[str] = Query(None),
     verification_method: Optional[str] = Query(None),
     category: Optional[str] = Query(None),
+    target_audience: Optional[str] = Query(None),
 ):
     """List all canonical controls, with optional filters."""
     query = f"""
@@ -313,6 +321,9 @@ async def list_controls(
     if category:
         query += " AND category = :cat"
         params["cat"] = category
+    if target_audience:
+        query += " AND target_audience = :ta"
+        params["ta"] = target_audience
 
     query += " ORDER BY control_id"
 
@@ -384,7 +395,7 @@ async def create_control(body: ControlCreateRequest):
                     open_anchors, release_state, tags,
                     license_rule, source_original_text, source_citation,
                     customer_visible, verification_method, category,
-                    generation_metadata
+                    target_audience, generation_metadata
                 ) VALUES (
                     :fw_id, :cid, :title, :objective, :rationale,
                     CAST(:scope AS jsonb), CAST(:requirements AS jsonb),
@@ -394,7 +405,7 @@ async def create_control(body: ControlCreateRequest):
                     :license_rule, :source_original_text,
                     CAST(:source_citation AS jsonb),
                     :customer_visible, :verification_method, :category,
-                    CAST(:generation_metadata AS jsonb)
+                    :target_audience, CAST(:generation_metadata AS jsonb)
                 )
                 RETURNING {_CONTROL_COLS}
             """),
@@ -421,6 +432,7 @@ async def create_control(body: ControlCreateRequest):
                 "customer_visible": body.customer_visible,
                 "verification_method": body.verification_method,
                 "category": body.category,
+                "target_audience": body.target_audience,
                 "generation_metadata": _json.dumps(body.generation_metadata) if body.generation_metadata else None,
             },
         ).fetchone()
@@ -647,6 +659,7 @@ def _control_row(r) -> dict:
         "customer_visible": r.customer_visible,
         "verification_method": r.verification_method,
         "category": r.category,
+        "target_audience": r.target_audience,
         "generation_metadata": r.generation_metadata,
         "created_at": r.created_at.isoformat() if r.created_at else None,
         "updated_at": r.updated_at.isoformat() if r.updated_at else None,

backend-compliance/compliance/api/control_generator_routes.py

@@ -12,6 +12,7 @@ Endpoints:
   POST /v1/canonical/blocked-sources/cleanup      — Start cleanup workflow
 """
 
+import asyncio
 import json
 import logging
 from typing import Optional, List
@@ -89,9 +90,42 @@ class BlockedSourceResponse(BaseModel):
 # ENDPOINTS
 # =============================================================================
 
+async def _run_pipeline_background(config: GeneratorConfig, job_id: str):
+    """Run the pipeline in the background. Uses its own DB session."""
+    db = SessionLocal()
+    try:
+        config.existing_job_id = job_id
+        pipeline = ControlGeneratorPipeline(db=db, rag_client=get_rag_client())
+        result = await pipeline.run(config)
+        logger.info(
+            "Background generation job %s completed: %d controls from %d chunks",
+            job_id, result.controls_generated, result.total_chunks_scanned,
+        )
+    except Exception as e:
+        logger.error("Background generation job %s failed: %s", job_id, e)
+        # Update job as failed
+        try:
+            db.execute(
+                text("""
+                    UPDATE canonical_generation_jobs
+                    SET status = 'failed', errors = :errors, completed_at = NOW()
+                    WHERE id = CAST(:job_id AS uuid)
+                """),
+                {"job_id": job_id, "errors": json.dumps([str(e)])},
+            )
+            db.commit()
+        except Exception:
+            pass
+    finally:
+        db.close()
+
+
 @router.post("/generate", response_model=GenerateResponse)
 async def start_generation(req: GenerateRequest):
-    """Start a control generation run."""
+    """Start a control generation run (runs in background).
+
+    Returns immediately with job_id. Use GET /generate/status/{job_id} to poll progress.
+    """
     config = GeneratorConfig(
         collections=req.collections,
         domain=req.domain,
@@ -101,30 +135,63 @@ async def start_generation(req: GenerateRequest):
         dry_run=req.dry_run,
     )
 
+    if req.dry_run:
+        # Dry run: execute synchronously and return controls
+        db = SessionLocal()
+        try:
+            pipeline = ControlGeneratorPipeline(db=db, rag_client=get_rag_client())
+            result = await pipeline.run(config)
+            return GenerateResponse(
+                job_id=result.job_id,
+                status=result.status,
+                message=f"Dry run: {result.controls_generated} controls from {result.total_chunks_scanned} chunks",
+                total_chunks_scanned=result.total_chunks_scanned,
+                controls_generated=result.controls_generated,
+                controls_verified=result.controls_verified,
+                controls_needs_review=result.controls_needs_review,
+                controls_too_close=result.controls_too_close,
+                controls_duplicates_found=result.controls_duplicates_found,
+                errors=result.errors,
+                controls=result.controls,
+            )
+        except Exception as e:
+            logger.error("Dry run failed: %s", e)
+            raise HTTPException(status_code=500, detail=str(e))
+        finally:
+            db.close()
+
+    # Create job record first so we can return the ID
     db = SessionLocal()
     try:
-        pipeline = ControlGeneratorPipeline(db=db, rag_client=get_rag_client())
-        result = await pipeline.run(config)
-
-        return GenerateResponse(
-            job_id=result.job_id,
-            status=result.status,
-            message=f"Generated {result.controls_generated} controls from {result.total_chunks_scanned} chunks",
-            total_chunks_scanned=result.total_chunks_scanned,
-            controls_generated=result.controls_generated,
-            controls_verified=result.controls_verified,
-            controls_needs_review=result.controls_needs_review,
-            controls_too_close=result.controls_too_close,
-            controls_duplicates_found=result.controls_duplicates_found,
-            errors=result.errors,
-            controls=result.controls if req.dry_run else [],
+        result = db.execute(
+            text("""
+                INSERT INTO canonical_generation_jobs (status, config)
+                VALUES ('running', :config)
+                RETURNING id
+            """),
+            {"config": json.dumps(config.model_dump())},
         )
+        db.commit()
+        row = result.fetchone()
+        job_id = str(row[0]) if row else None
     except Exception as e:
-        logger.error("Generation failed: %s", e)
-        raise HTTPException(status_code=500, detail=str(e))
+        logger.error("Failed to create job: %s", e)
+        raise HTTPException(status_code=500, detail=f"Failed to create job: {e}")
     finally:
         db.close()
 
+    if not job_id:
+        raise HTTPException(status_code=500, detail="Failed to create job record")
+
+    # Launch pipeline in background
+    asyncio.create_task(_run_pipeline_background(config, job_id))
+
+    return GenerateResponse(
+        job_id=job_id,
+        status="running",
+        message="Generation started in background. Poll /generate/status/{job_id} for progress.",
+    )
+
 
 @router.get("/generate/status/{job_id}")
 async def get_job_status(job_id: str):

backend-compliance/compliance/api/dashboard_routes.py

@@ -5,16 +5,23 @@ Endpoints:
 - /dashboard: Main compliance dashboard
 - /dashboard/executive: Executive summary for managers
 - /dashboard/trend: Compliance score trend over time
+- /dashboard/roadmap: Prioritised controls in 4 buckets
+- /dashboard/module-status: Completion status of each SDK module
+- /dashboard/next-actions: Top 5 most important actions
+- /dashboard/snapshot: Save / query compliance score snapshots
 - /score: Quick compliance score
 - /reports: Report generation
 """
 
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime, date, timedelta
 from calendar import month_abbr
-from typing import Optional
+from typing import Optional, Dict, Any, List
+from decimal import Decimal
 
 from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
+from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from classroom_engine.database import get_db
@@ -34,6 +41,8 @@ from .schemas import (
     DeadlineItem,
     TeamWorkloadItem,
 )
+from .tenant_utils import get_tenant_id as _get_tenant_id
+from .db_utils import row_to_dict as _row_to_dict
 
 logger = logging.getLogger(__name__)
 router = APIRouter(tags=["compliance-dashboard"])
@@ -322,6 +331,272 @@ async def get_compliance_trend(
     }
 
 
+# ============================================================================
+# Dashboard Extended — Roadmap, Module-Status, Next-Actions, Snapshots
+# ============================================================================
+
+# Weight map for control prioritisation
+_PRIORITY_WEIGHTS = {"legal": 5, "security": 3, "best_practice": 1, "operational": 2}
+
+# SDK module definitions → DB table used for counting completion
+_MODULE_DEFS: List[Dict[str, str]] = [
+    {"key": "vvt", "label": "VVT", "table": "compliance_vvt_activities"},
+    {"key": "tom", "label": "TOM", "table": "compliance_toms"},
+    {"key": "dsfa", "label": "DSFA", "table": "compliance_dsfa_assessments"},
+    {"key": "loeschfristen", "label": "Loeschfristen", "table": "compliance_loeschfristen"},
+    {"key": "risks", "label": "Risiken", "table": "compliance_risks"},
+    {"key": "controls", "label": "Controls", "table": "compliance_controls"},
+    {"key": "evidence", "label": "Nachweise", "table": "compliance_evidence"},
+    {"key": "obligations", "label": "Pflichten", "table": "compliance_obligations"},
+    {"key": "incidents", "label": "Vorfaelle", "table": "compliance_notfallplan_incidents"},
+    {"key": "vendor", "label": "Auftragsverarbeiter", "table": "compliance_vendor_assessments"},
+    {"key": "legal_templates", "label": "Rechtl. Dokumente", "table": "compliance_legal_templates"},
+    {"key": "training", "label": "Schulungen", "table": "training_modules"},
+    {"key": "audit", "label": "Audit", "table": "compliance_audit_sessions"},
+    {"key": "security_backlog", "label": "Security-Backlog", "table": "compliance_security_backlog"},
+    {"key": "quality", "label": "Qualitaet", "table": "compliance_quality_items"},
+]
+
+
+@router.get("/dashboard/roadmap")
+async def get_dashboard_roadmap(
+    db: Session = Depends(get_db),
+    tenant_id: str = Depends(_get_tenant_id),
+):
+    """Prioritised controls in 4 buckets: Quick Wins, Must Have, Should Have, Nice to Have."""
+    ctrl_repo = ControlRepository(db)
+    controls = ctrl_repo.get_all()
+    today = datetime.utcnow().date()
+
+    buckets: Dict[str, list] = {
+        "quick_wins": [],
+        "must_have": [],
+        "should_have": [],
+        "nice_to_have": [],
+    }
+
+    for ctrl in controls:
+        status = ctrl.status.value if ctrl.status else "planned"
+        if status == "pass":
+            continue  # already done
+
+        weight = _PRIORITY_WEIGHTS.get(ctrl.category if hasattr(ctrl, "category") else "best_practice", 1)
+        days_overdue = 0
+        if ctrl.next_review_at:
+            review_date = ctrl.next_review_at.date() if hasattr(ctrl.next_review_at, "date") else ctrl.next_review_at
+            days_overdue = (today - review_date).days
+
+        urgency = weight * 2 + (1 if days_overdue > 0 else 0)
+
+        item = {
+            "id": str(ctrl.id),
+            "control_id": ctrl.control_id,
+            "title": ctrl.title,
+            "status": status,
+            "domain": ctrl.domain.value if ctrl.domain else "unknown",
+            "owner": ctrl.owner,
+            "next_review_at": ctrl.next_review_at.isoformat() if ctrl.next_review_at else None,
+            "days_overdue": max(0, days_overdue),
+            "weight": weight,
+        }
+
+        if weight >= 5 and days_overdue > 0:
+            buckets["quick_wins"].append(item)
+        elif weight >= 4:
+            buckets["must_have"].append(item)
+        elif weight >= 2:
+            buckets["should_have"].append(item)
+        else:
+            buckets["nice_to_have"].append(item)
+
+    # Sort each bucket by urgency desc
+    for key in buckets:
+        buckets[key].sort(key=lambda x: x["days_overdue"], reverse=True)
+
+    return {
+        "buckets": buckets,
+        "counts": {k: len(v) for k, v in buckets.items()},
+        "generated_at": datetime.utcnow().isoformat(),
+    }
+
+
+@router.get("/dashboard/module-status")
+async def get_module_status(
+    db: Session = Depends(get_db),
+    tenant_id: str = Depends(_get_tenant_id),
+):
+    """Completion status for each SDK module based on DB record counts."""
+    modules = []
+    for mod in _MODULE_DEFS:
+        try:
+            row = db.execute(
+                text(f"SELECT COUNT(*) FROM {mod['table']} WHERE tenant_id = :tid"),
+                {"tid": tenant_id},
+            ).fetchone()
+            count = int(row[0]) if row else 0
+        except Exception:
+            count = 0
+
+        # Simple heuristic: 0 = not started, 1-2 = in progress, 3+ = complete
+        if count == 0:
+            status = "not_started"
+            progress = 0
+        elif count < 3:
+            status = "in_progress"
+            progress = min(60, count * 30)
+        else:
+            status = "complete"
+            progress = 100
+
+        modules.append({
+            "key": mod["key"],
+            "label": mod["label"],
+            "count": count,
+            "status": status,
+            "progress": progress,
+        })
+
+    started = sum(1 for m in modules if m["status"] != "not_started")
+    complete = sum(1 for m in modules if m["status"] == "complete")
+
+    return {
+        "modules": modules,
+        "total": len(modules),
+        "started": started,
+        "complete": complete,
+        "overall_progress": round((complete / len(modules)) * 100, 1) if modules else 0,
+    }
+
+
+@router.get("/dashboard/next-actions")
+async def get_next_actions(
+    limit: int = Query(5, ge=1, le=20),
+    db: Session = Depends(get_db),
+    tenant_id: str = Depends(_get_tenant_id),
+):
+    """Top N most important actions sorted by urgency*impact."""
+    ctrl_repo = ControlRepository(db)
+    controls = ctrl_repo.get_all()
+    today = datetime.utcnow().date()
+
+    actions = []
+    for ctrl in controls:
+        status = ctrl.status.value if ctrl.status else "planned"
+        if status == "pass":
+            continue
+
+        days_overdue = 0
+        if ctrl.next_review_at:
+            review_date = ctrl.next_review_at.date() if hasattr(ctrl.next_review_at, "date") else ctrl.next_review_at
+            days_overdue = max(0, (today - review_date).days)
+
+        weight = _PRIORITY_WEIGHTS.get(ctrl.category if hasattr(ctrl, "category") else "best_practice", 1)
+        urgency_score = weight * 10 + days_overdue
+
+        actions.append({
+            "id": str(ctrl.id),
+            "control_id": ctrl.control_id,
+            "title": ctrl.title,
+            "status": status,
+            "domain": ctrl.domain.value if ctrl.domain else "unknown",
+            "owner": ctrl.owner,
+            "days_overdue": days_overdue,
+            "urgency_score": urgency_score,
+            "reason": "Ueberfaellig" if days_overdue > 0 else "Offen",
+        })
+
+    actions.sort(key=lambda x: x["urgency_score"], reverse=True)
+    return {"actions": actions[:limit]}
+
+
+@router.post("/dashboard/snapshot")
+async def create_score_snapshot(
+    db: Session = Depends(get_db),
+    tenant_id: str = Depends(_get_tenant_id),
+):
+    """Save current compliance score as a historical snapshot."""
+    ctrl_repo = ControlRepository(db)
+    evidence_repo = EvidenceRepository(db)
+    risk_repo = RiskRepository(db)
+
+    ctrl_stats = ctrl_repo.get_statistics()
+    evidence_stats = evidence_repo.get_statistics()
+    risks = risk_repo.get_all()
+
+    total = ctrl_stats.get("total", 0)
+    passing = ctrl_stats.get("pass", 0)
+    partial = ctrl_stats.get("partial", 0)
+    score = round(((passing + partial * 0.5) / total) * 100, 2) if total > 0 else 0
+
+    risks_high = sum(1 for r in risks if (r.inherent_risk.value if r.inherent_risk else "low") in ("high", "critical"))
+
+    today = date.today()
+
+    row = db.execute(text("""
+        INSERT INTO compliance_score_snapshots (
+            tenant_id, score, controls_total, controls_pass, controls_partial,
+            evidence_total, evidence_valid, risks_total, risks_high, snapshot_date
+        ) VALUES (
+            :tenant_id, :score, :controls_total, :controls_pass, :controls_partial,
+            :evidence_total, :evidence_valid, :risks_total, :risks_high, :snapshot_date
+        )
+        ON CONFLICT (tenant_id, project_id, snapshot_date) DO UPDATE SET
+            score = EXCLUDED.score,
+            controls_total = EXCLUDED.controls_total,
+            controls_pass = EXCLUDED.controls_pass,
+            controls_partial = EXCLUDED.controls_partial,
+            evidence_total = EXCLUDED.evidence_total,
+            evidence_valid = EXCLUDED.evidence_valid,
+            risks_total = EXCLUDED.risks_total,
+            risks_high = EXCLUDED.risks_high
+        RETURNING *
+    """), {
+        "tenant_id": tenant_id,
+        "score": score,
+        "controls_total": total,
+        "controls_pass": passing,
+        "controls_partial": partial,
+        "evidence_total": evidence_stats.get("total", 0),
+        "evidence_valid": evidence_stats.get("by_status", {}).get("valid", 0),
+        "risks_total": len(risks),
+        "risks_high": risks_high,
+        "snapshot_date": today,
+    }).fetchone()
+    db.commit()
+
+    return _row_to_dict(row)
+
+
+@router.get("/dashboard/score-history")
+async def get_score_history(
+    months: int = Query(12, ge=1, le=36),
+    db: Session = Depends(get_db),
+    tenant_id: str = Depends(_get_tenant_id),
+):
+    """Get compliance score history from snapshots."""
+    since = date.today() - timedelta(days=months * 30)
+
+    rows = db.execute(text("""
+        SELECT * FROM compliance_score_snapshots
+        WHERE tenant_id = :tenant_id AND snapshot_date >= :since
+        ORDER BY snapshot_date ASC
+    """), {"tenant_id": tenant_id, "since": since}).fetchall()
+
+    snapshots = []
+    for r in rows:
+        d = _row_to_dict(r)
+        # Convert Decimal to float for JSON
+        if isinstance(d.get("score"), Decimal):
+            d["score"] = float(d["score"])
+        snapshots.append(d)
+
+    return {
+        "snapshots": snapshots,
+        "total": len(snapshots),
+        "period_months": months,
+    }
+
+
 # ============================================================================
 # Reports
 # ============================================================================

backend-compliance/compliance/api/legal_template_routes.py

@@ -50,6 +50,14 @@ VALID_DOCUMENT_TYPES = {
     "cookie_banner",
     "agb",
     "clause",
+    # Security document templates (Migration 051)
+    "it_security_concept",
+    "data_protection_concept",
+    "backup_recovery_concept",
+    "logging_concept",
+    "incident_response_plan",
+    "access_control_concept",
+    "risk_management_concept",
 }
 VALID_STATUSES = {"published", "draft", "archived"}