feat(multi-layer): complete Multi-Layer Control Architecture (Phases 1-8 + Pass 0)

Implements the full Multi-Layer Control Architecture for migrating ~25,000
Rich Controls into atomic, deduplicated Master Controls with full traceability.

Architecture: Legal Source → Obligation → Control Pattern → Master Control → Customer Instance

New services:
- ObligationExtractor: 3-tier extraction (exact → embedding → LLM)
- PatternMatcher: 2-tier matching (keyword + embedding + domain-bonus)
- ControlComposer: Pattern + Obligation → Master Control
- PipelineAdapter: Pipeline integration + Migration Passes 1-5
- DecompositionPass: Pass 0a/0b — Rich Control → atomic Controls
- CrosswalkRoutes: 15 API endpoints under /v1/canonical/

New DB schema:
- Migration 060: obligation_extractions, control_patterns, crosswalk_matrix
- Migration 061: obligation_candidates, parent_control_uuid tracking

Pattern Library: 50 YAML patterns (30 core + 20 IT-security)
Go SDK: Pattern loader with YAML validation and indexing
Documentation: MkDocs updated with full architecture overview

500 Python tests passing across all components.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

backend-compliance/compliance/api/__init__.py

@@ -53,6 +53,7 @@ _ROUTER_MODULES = [
     "wiki_routes",
     "canonical_control_routes",
     "control_generator_routes",
+    "crosswalk_routes",
     "process_task_routes",
     "evidence_check_routes",
 ]

backend-compliance/compliance/api/crosswalk_routes.py

@@ -0,0 +1,623 @@
+"""
+FastAPI routes for the Multi-Layer Control Architecture.
+
+Pattern Library, Obligation Extraction, Crosswalk Matrix, and Migration endpoints.
+
+Endpoints:
+  GET  /v1/canonical/patterns                          — All patterns (with filters)
+  GET  /v1/canonical/patterns/{pattern_id}             — Single pattern
+  GET  /v1/canonical/patterns/{pattern_id}/controls    — Controls for a pattern
+
+  POST /v1/canonical/obligations/extract               — Extract obligations from text
+  GET  /v1/canonical/crosswalk                         — Query crosswalk matrix
+  GET  /v1/canonical/crosswalk/stats                   — Coverage statistics
+
+  POST /v1/canonical/migrate/decompose                 — Pass 0a: Obligation extraction
+  POST /v1/canonical/migrate/compose-atomic            — Pass 0b: Atomic control composition
+  POST /v1/canonical/migrate/link-obligations          — Pass 1: Obligation linkage
+  POST /v1/canonical/migrate/classify-patterns         — Pass 2: Pattern classification
+  POST /v1/canonical/migrate/triage                    — Pass 3: Quality triage
+  POST /v1/canonical/migrate/backfill-crosswalk        — Pass 4: Crosswalk backfill
+  POST /v1/canonical/migrate/deduplicate               — Pass 5: Deduplication
+  GET  /v1/canonical/migrate/status                    — Migration progress
+  GET  /v1/canonical/migrate/decomposition-status      — Decomposition progress
+"""
+
+import json
+import logging
+from typing import Optional, List
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel
+from sqlalchemy import text
+
+from database import SessionLocal
+
+logger = logging.getLogger(__name__)
+router = APIRouter(prefix="/v1/canonical", tags=["crosswalk"])
+
+
+# =============================================================================
+# REQUEST / RESPONSE MODELS
+# =============================================================================
+
+
+class PatternResponse(BaseModel):
+    id: str
+    name: str
+    name_de: str
+    domain: str
+    category: str
+    description: str
+    objective_template: str
+    severity_default: str
+    implementation_effort_default: str = "m"
+    tags: list = []
+    composable_with: list = []
+    open_anchor_refs: list = []
+    controls_count: int = 0
+
+
+class PatternListResponse(BaseModel):
+    patterns: List[PatternResponse]
+    total: int
+
+
+class PatternDetailResponse(PatternResponse):
+    rationale_template: str = ""
+    requirements_template: list = []
+    test_procedure_template: list = []
+    evidence_template: list = []
+    obligation_match_keywords: list = []
+
+
+class ObligationExtractRequest(BaseModel):
+    text: str
+    regulation_code: Optional[str] = None
+    article: Optional[str] = None
+    paragraph: Optional[str] = None
+
+
+class ObligationExtractResponse(BaseModel):
+    obligation_id: Optional[str] = None
+    obligation_title: Optional[str] = None
+    obligation_text: Optional[str] = None
+    method: str = "none"
+    confidence: float = 0.0
+    regulation_id: Optional[str] = None
+    pattern_id: Optional[str] = None
+    pattern_confidence: float = 0.0
+
+
+class CrosswalkRow(BaseModel):
+    regulation_code: str = ""
+    article: Optional[str] = None
+    obligation_id: Optional[str] = None
+    pattern_id: Optional[str] = None
+    master_control_id: Optional[str] = None
+    confidence: float = 0.0
+    source: str = "auto"
+
+
+class CrosswalkQueryResponse(BaseModel):
+    rows: List[CrosswalkRow]
+    total: int
+
+
+class CrosswalkStatsResponse(BaseModel):
+    total_rows: int = 0
+    regulations_covered: int = 0
+    obligations_linked: int = 0
+    patterns_used: int = 0
+    controls_linked: int = 0
+    coverage_by_regulation: dict = {}
+
+
+class MigrationRequest(BaseModel):
+    limit: int = 0  # 0 = no limit
+
+
+class MigrationResponse(BaseModel):
+    status: str = "completed"
+    stats: dict = {}
+
+
+class MigrationStatusResponse(BaseModel):
+    total_controls: int = 0
+    has_obligation: int = 0
+    has_pattern: int = 0
+    fully_linked: int = 0
+    deprecated: int = 0
+    coverage_obligation_pct: float = 0.0
+    coverage_pattern_pct: float = 0.0
+    coverage_full_pct: float = 0.0
+
+
+class DecompositionStatusResponse(BaseModel):
+    rich_controls: int = 0
+    decomposed_controls: int = 0
+    total_candidates: int = 0
+    validated: int = 0
+    rejected: int = 0
+    composed: int = 0
+    atomic_controls: int = 0
+    decomposition_pct: float = 0.0
+    composition_pct: float = 0.0
+
+
+# =============================================================================
+# PATTERN LIBRARY ENDPOINTS
+# =============================================================================
+
+
+@router.get("/patterns", response_model=PatternListResponse)
+async def list_patterns(
+    domain: Optional[str] = Query(None, description="Filter by domain (e.g. AUTH, CRYP)"),
+    category: Optional[str] = Query(None, description="Filter by category"),
+    tag: Optional[str] = Query(None, description="Filter by tag"),
+):
+    """List all control patterns with optional filters."""
+    from compliance.services.pattern_matcher import PatternMatcher
+
+    matcher = PatternMatcher()
+    matcher._load_patterns()
+    matcher._build_keyword_index()
+
+    patterns = matcher._patterns
+
+    if domain:
+        patterns = [p for p in patterns if p.domain == domain.upper()]
+    if category:
+        patterns = [p for p in patterns if p.category == category.lower()]
+    if tag:
+        patterns = [p for p in patterns if tag.lower() in [t.lower() for t in p.tags]]
+
+    # Count controls per pattern from DB
+    control_counts = _get_pattern_control_counts()
+
+    response_patterns = []
+    for p in patterns:
+        response_patterns.append(PatternResponse(
+            id=p.id,
+            name=p.name,
+            name_de=p.name_de,
+            domain=p.domain,
+            category=p.category,
+            description=p.description,
+            objective_template=p.objective_template,
+            severity_default=p.severity_default,
+            implementation_effort_default=p.implementation_effort_default,
+            tags=p.tags,
+            composable_with=p.composable_with,
+            open_anchor_refs=p.open_anchor_refs,
+            controls_count=control_counts.get(p.id, 0),
+        ))
+
+    return PatternListResponse(patterns=response_patterns, total=len(response_patterns))
+
+
+@router.get("/patterns/{pattern_id}", response_model=PatternDetailResponse)
+async def get_pattern(pattern_id: str):
+    """Get a single control pattern by ID."""
+    from compliance.services.pattern_matcher import PatternMatcher
+
+    matcher = PatternMatcher()
+    matcher._load_patterns()
+
+    pattern = matcher.get_pattern(pattern_id)
+    if not pattern:
+        raise HTTPException(status_code=404, detail=f"Pattern {pattern_id} not found")
+
+    control_counts = _get_pattern_control_counts()
+
+    return PatternDetailResponse(
+        id=pattern.id,
+        name=pattern.name,
+        name_de=pattern.name_de,
+        domain=pattern.domain,
+        category=pattern.category,
+        description=pattern.description,
+        objective_template=pattern.objective_template,
+        rationale_template=pattern.rationale_template,
+        requirements_template=pattern.requirements_template,
+        test_procedure_template=pattern.test_procedure_template,
+        evidence_template=pattern.evidence_template,
+        severity_default=pattern.severity_default,
+        implementation_effort_default=pattern.implementation_effort_default,
+        tags=pattern.tags,
+        composable_with=pattern.composable_with,
+        open_anchor_refs=pattern.open_anchor_refs,
+        obligation_match_keywords=pattern.obligation_match_keywords,
+        controls_count=control_counts.get(pattern.id, 0),
+    )
+
+
+@router.get("/patterns/{pattern_id}/controls")
+async def get_pattern_controls(
+    pattern_id: str,
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+):
+    """Get controls generated from a specific pattern."""
+    db = SessionLocal()
+    try:
+        result = db.execute(
+            text("""
+                SELECT id, control_id, title, objective, severity,
+                       release_state, category, obligation_ids
+                FROM canonical_controls
+                WHERE pattern_id = :pattern_id
+                  AND release_state NOT IN ('deprecated')
+                ORDER BY control_id
+                LIMIT :limit OFFSET :offset
+            """),
+            {"pattern_id": pattern_id.upper(), "limit": limit, "offset": offset},
+        )
+        rows = result.fetchall()
+
+        count_result = db.execute(
+            text("""
+                SELECT count(*) FROM canonical_controls
+                WHERE pattern_id = :pattern_id
+                  AND release_state NOT IN ('deprecated')
+            """),
+            {"pattern_id": pattern_id.upper()},
+        )
+        total = count_result.fetchone()[0]
+
+        controls = []
+        for row in rows:
+            obl_ids = row[7]
+            if isinstance(obl_ids, str):
+                try:
+                    obl_ids = json.loads(obl_ids)
+                except (json.JSONDecodeError, TypeError):
+                    obl_ids = []
+            controls.append({
+                "id": str(row[0]),
+                "control_id": row[1],
+                "title": row[2],
+                "objective": row[3],
+                "severity": row[4],
+                "release_state": row[5],
+                "category": row[6],
+                "obligation_ids": obl_ids or [],
+            })
+
+        return {"controls": controls, "total": total}
+    finally:
+        db.close()
+
+
+# =============================================================================
+# OBLIGATION EXTRACTION ENDPOINT
+# =============================================================================
+
+
+@router.post("/obligations/extract", response_model=ObligationExtractResponse)
+async def extract_obligation(req: ObligationExtractRequest):
+    """Extract obligation from text using 3-tier strategy, then match to pattern."""
+    from compliance.services.obligation_extractor import ObligationExtractor
+    from compliance.services.pattern_matcher import PatternMatcher
+
+    extractor = ObligationExtractor()
+    await extractor.initialize()
+
+    obligation = await extractor.extract(
+        chunk_text=req.text,
+        regulation_code=req.regulation_code or "",
+        article=req.article,
+        paragraph=req.paragraph,
+    )
+
+    # Also match to pattern
+    matcher = PatternMatcher()
+    matcher._load_patterns()
+    matcher._build_keyword_index()
+
+    pattern_text = obligation.obligation_text or obligation.obligation_title or req.text[:500]
+    pattern_result = matcher._tier1_keyword(pattern_text, obligation.regulation_id)
+
+    return ObligationExtractResponse(
+        obligation_id=obligation.obligation_id,
+        obligation_title=obligation.obligation_title,
+        obligation_text=obligation.obligation_text,
+        method=obligation.method,
+        confidence=obligation.confidence,
+        regulation_id=obligation.regulation_id,
+        pattern_id=pattern_result.pattern_id if pattern_result else None,
+        pattern_confidence=pattern_result.confidence if pattern_result else 0,
+    )
+
+
+# =============================================================================
+# CROSSWALK MATRIX ENDPOINTS
+# =============================================================================
+
+
+@router.get("/crosswalk", response_model=CrosswalkQueryResponse)
+async def query_crosswalk(
+    regulation_code: Optional[str] = Query(None),
+    article: Optional[str] = Query(None),
+    obligation_id: Optional[str] = Query(None),
+    pattern_id: Optional[str] = Query(None),
+    limit: int = Query(100, ge=1, le=1000),
+    offset: int = Query(0, ge=0),
+):
+    """Query the crosswalk matrix with filters."""
+    db = SessionLocal()
+    try:
+        conditions = ["1=1"]
+        params = {"limit": limit, "offset": offset}
+
+        if regulation_code:
+            conditions.append("regulation_code = :reg")
+            params["reg"] = regulation_code
+        if article:
+            conditions.append("article = :art")
+            params["art"] = article
+        if obligation_id:
+            conditions.append("obligation_id = :obl")
+            params["obl"] = obligation_id
+        if pattern_id:
+            conditions.append("pattern_id = :pat")
+            params["pat"] = pattern_id
+
+        where = " AND ".join(conditions)
+
+        result = db.execute(
+            text(f"""
+                SELECT regulation_code, article, obligation_id,
+                       pattern_id, master_control_id, confidence, source
+                FROM crosswalk_matrix
+                WHERE {where}
+                ORDER BY regulation_code, article
+                LIMIT :limit OFFSET :offset
+            """),
+            params,
+        )
+        rows = result.fetchall()
+
+        count_result = db.execute(
+            text(f"SELECT count(*) FROM crosswalk_matrix WHERE {where}"),
+            params,
+        )
+        total = count_result.fetchone()[0]
+
+        crosswalk_rows = [
+            CrosswalkRow(
+                regulation_code=r[0] or "",
+                article=r[1],
+                obligation_id=r[2],
+                pattern_id=r[3],
+                master_control_id=r[4],
+                confidence=float(r[5] or 0),
+                source=r[6] or "auto",
+            )
+            for r in rows
+        ]
+
+        return CrosswalkQueryResponse(rows=crosswalk_rows, total=total)
+    finally:
+        db.close()
+
+
+@router.get("/crosswalk/stats", response_model=CrosswalkStatsResponse)
+async def crosswalk_stats():
+    """Get crosswalk coverage statistics."""
+    db = SessionLocal()
+    try:
+        row = db.execute(text("""
+            SELECT
+                count(*) AS total,
+                count(DISTINCT regulation_code) FILTER (WHERE regulation_code != '') AS regs,
+                count(DISTINCT obligation_id) FILTER (WHERE obligation_id IS NOT NULL) AS obls,
+                count(DISTINCT pattern_id) FILTER (WHERE pattern_id IS NOT NULL) AS pats,
+                count(DISTINCT master_control_id) FILTER (WHERE master_control_id IS NOT NULL) AS ctrls
+            FROM crosswalk_matrix
+        """)).fetchone()
+
+        # Coverage by regulation
+        reg_rows = db.execute(text("""
+            SELECT regulation_code, count(*) AS cnt
+            FROM crosswalk_matrix
+            WHERE regulation_code != ''
+            GROUP BY regulation_code
+            ORDER BY cnt DESC
+        """)).fetchall()
+
+        coverage = {r[0]: r[1] for r in reg_rows}
+
+        return CrosswalkStatsResponse(
+            total_rows=row[0],
+            regulations_covered=row[1],
+            obligations_linked=row[2],
+            patterns_used=row[3],
+            controls_linked=row[4],
+            coverage_by_regulation=coverage,
+        )
+    finally:
+        db.close()
+
+
+# =============================================================================
+# MIGRATION ENDPOINTS
+# =============================================================================
+
+
+@router.post("/migrate/decompose", response_model=MigrationResponse)
+async def migrate_decompose(req: MigrationRequest):
+    """Pass 0a: Extract obligation candidates from rich controls."""
+    from compliance.services.decomposition_pass import DecompositionPass
+
+    db = SessionLocal()
+    try:
+        decomp = DecompositionPass(db=db)
+        stats = await decomp.run_pass0a(limit=req.limit)
+        return MigrationResponse(status="completed", stats=stats)
+    except Exception as e:
+        logger.error("Decomposition pass 0a failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.post("/migrate/compose-atomic", response_model=MigrationResponse)
+async def migrate_compose_atomic(req: MigrationRequest):
+    """Pass 0b: Compose atomic controls from obligation candidates."""
+    from compliance.services.decomposition_pass import DecompositionPass
+
+    db = SessionLocal()
+    try:
+        decomp = DecompositionPass(db=db)
+        stats = await decomp.run_pass0b(limit=req.limit)
+        return MigrationResponse(status="completed", stats=stats)
+    except Exception as e:
+        logger.error("Decomposition pass 0b failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.post("/migrate/link-obligations", response_model=MigrationResponse)
+async def migrate_link_obligations(req: MigrationRequest):
+    """Pass 1: Link controls to obligations via source_citation article."""
+    from compliance.services.pipeline_adapter import MigrationPasses
+
+    db = SessionLocal()
+    try:
+        migration = MigrationPasses(db=db)
+        await migration.initialize()
+        stats = await migration.run_pass1_obligation_linkage(limit=req.limit)
+        return MigrationResponse(status="completed", stats=stats)
+    except Exception as e:
+        logger.error("Migration pass 1 failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.post("/migrate/classify-patterns", response_model=MigrationResponse)
+async def migrate_classify_patterns(req: MigrationRequest):
+    """Pass 2: Classify controls into patterns via keyword matching."""
+    from compliance.services.pipeline_adapter import MigrationPasses
+
+    db = SessionLocal()
+    try:
+        migration = MigrationPasses(db=db)
+        await migration.initialize()
+        stats = await migration.run_pass2_pattern_classification(limit=req.limit)
+        return MigrationResponse(status="completed", stats=stats)
+    except Exception as e:
+        logger.error("Migration pass 2 failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.post("/migrate/triage", response_model=MigrationResponse)
+async def migrate_triage():
+    """Pass 3: Quality triage — categorize by linkage completeness."""
+    from compliance.services.pipeline_adapter import MigrationPasses
+
+    db = SessionLocal()
+    try:
+        migration = MigrationPasses(db=db)
+        stats = migration.run_pass3_quality_triage()
+        return MigrationResponse(status="completed", stats=stats)
+    except Exception as e:
+        logger.error("Migration pass 3 failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.post("/migrate/backfill-crosswalk", response_model=MigrationResponse)
+async def migrate_backfill_crosswalk():
+    """Pass 4: Create crosswalk rows for linked controls."""
+    from compliance.services.pipeline_adapter import MigrationPasses
+
+    db = SessionLocal()
+    try:
+        migration = MigrationPasses(db=db)
+        stats = migration.run_pass4_crosswalk_backfill()
+        return MigrationResponse(status="completed", stats=stats)
+    except Exception as e:
+        logger.error("Migration pass 4 failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.post("/migrate/deduplicate", response_model=MigrationResponse)
+async def migrate_deduplicate():
+    """Pass 5: Mark duplicate controls (same obligation + pattern)."""
+    from compliance.services.pipeline_adapter import MigrationPasses
+
+    db = SessionLocal()
+    try:
+        migration = MigrationPasses(db=db)
+        stats = migration.run_pass5_deduplication()
+        return MigrationResponse(status="completed", stats=stats)
+    except Exception as e:
+        logger.error("Migration pass 5 failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.get("/migrate/status", response_model=MigrationStatusResponse)
+async def migration_status():
+    """Get overall migration progress."""
+    from compliance.services.pipeline_adapter import MigrationPasses
+
+    db = SessionLocal()
+    try:
+        migration = MigrationPasses(db=db)
+        status = migration.migration_status()
+        return MigrationStatusResponse(**status)
+    except Exception as e:
+        logger.error("Migration status failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+@router.get("/migrate/decomposition-status", response_model=DecompositionStatusResponse)
+async def decomposition_status():
+    """Get decomposition progress (Pass 0a/0b)."""
+    from compliance.services.decomposition_pass import DecompositionPass
+
+    db = SessionLocal()
+    try:
+        decomp = DecompositionPass(db=db)
+        status = decomp.decomposition_status()
+        return DecompositionStatusResponse(**status)
+    except Exception as e:
+        logger.error("Decomposition status failed: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        db.close()
+
+
+# =============================================================================
+# HELPERS
+# =============================================================================
+
+
+def _get_pattern_control_counts() -> dict[str, int]:
+    """Get count of controls per pattern_id from DB."""
+    db = SessionLocal()
+    try:
+        result = db.execute(text("""
+            SELECT pattern_id, count(*) AS cnt
+            FROM canonical_controls
+            WHERE pattern_id IS NOT NULL AND pattern_id != ''
+              AND release_state NOT IN ('deprecated')
+            GROUP BY pattern_id
+        """))
+        return {row[0]: row[1] for row in result.fetchall()}
+    except Exception:
+        return {}
+    finally:
+        db.close()