feat(sdk): VVT master libraries, process templates, Loeschfristen profiling + document

VVT: Master library tables (7 catalogs), 500+ seed entries, process templates
with instantiation, library API endpoints + 18 tests.
Loeschfristen: Baseline catalog, compliance checks, profiling engine, HTML document
generator, MkDocs documentation.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-19 11:56:25 +01:00
parent f2819b99af
commit 2a70441eaa
20 changed files with 6621 additions and 9 deletions

View File

@@ -1755,6 +1755,20 @@ class VVTActivityCreate(BaseModel):
next_review_at: Optional[datetime] = None
created_by: Optional[str] = None
dsfa_id: Optional[str] = None
# Library refs (optional, parallel to freetext)
purpose_refs: Optional[List[str]] = None
legal_basis_refs: Optional[List[str]] = None
data_subject_refs: Optional[List[str]] = None
data_category_refs: Optional[List[str]] = None
recipient_refs: Optional[List[str]] = None
retention_rule_ref: Optional[str] = None
transfer_mechanism_refs: Optional[List[str]] = None
tom_refs: Optional[List[str]] = None
source_template_id: Optional[str] = None
risk_score: Optional[int] = None
linked_loeschfristen_ids: Optional[List[str]] = None
linked_tom_measure_ids: Optional[List[str]] = None
art30_completeness: Optional[Dict[str, Any]] = None
class VVTActivityUpdate(BaseModel):
@@ -1783,6 +1797,20 @@ class VVTActivityUpdate(BaseModel):
next_review_at: Optional[datetime] = None
created_by: Optional[str] = None
dsfa_id: Optional[str] = None
# Library refs
purpose_refs: Optional[List[str]] = None
legal_basis_refs: Optional[List[str]] = None
data_subject_refs: Optional[List[str]] = None
data_category_refs: Optional[List[str]] = None
recipient_refs: Optional[List[str]] = None
retention_rule_ref: Optional[str] = None
transfer_mechanism_refs: Optional[List[str]] = None
tom_refs: Optional[List[str]] = None
source_template_id: Optional[str] = None
risk_score: Optional[int] = None
linked_loeschfristen_ids: Optional[List[str]] = None
linked_tom_measure_ids: Optional[List[str]] = None
art30_completeness: Optional[Dict[str, Any]] = None
class VVTActivityResponse(BaseModel):
@@ -1813,6 +1841,20 @@ class VVTActivityResponse(BaseModel):
next_review_at: Optional[datetime] = None
created_by: Optional[str] = None
dsfa_id: Optional[str] = None
# Library refs
purpose_refs: Optional[List[str]] = None
legal_basis_refs: Optional[List[str]] = None
data_subject_refs: Optional[List[str]] = None
data_category_refs: Optional[List[str]] = None
recipient_refs: Optional[List[str]] = None
retention_rule_ref: Optional[str] = None
transfer_mechanism_refs: Optional[List[str]] = None
tom_refs: Optional[List[str]] = None
source_template_id: Optional[str] = None
risk_score: Optional[int] = None
linked_loeschfristen_ids: Optional[List[str]] = None
linked_tom_measure_ids: Optional[List[str]] = None
art30_completeness: Optional[Dict[str, Any]] = None
created_at: datetime
updated_at: Optional[datetime] = None

View File

@@ -0,0 +1,427 @@
"""
FastAPI routes for VVT Master Libraries + Process Templates.
Library endpoints (read-only, global):
GET /vvt/libraries — Overview: all library types + counts
GET /vvt/libraries/data-subjects — Data subjects (filter: typical_for)
GET /vvt/libraries/data-categories — Hierarchical (filter: parent_id, is_art9, flat)
GET /vvt/libraries/recipients — Recipients (filter: type)
GET /vvt/libraries/legal-bases — Legal bases (filter: is_art9, type)
GET /vvt/libraries/retention-rules — Retention rules
GET /vvt/libraries/transfer-mechanisms — Transfer mechanisms
GET /vvt/libraries/purposes — Purposes (filter: typical_for)
GET /vvt/libraries/toms — TOMs (filter: category)
Template endpoints:
GET /vvt/templates — List templates (filter: business_function, search)
GET /vvt/templates/{id} — Single template with resolved labels
POST /vvt/templates/{id}/instantiate — Create VVT activity from template
"""
import logging
import uuid
from datetime import datetime
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.orm import Session
from classroom_engine.database import get_db
from ..db.vvt_library_models import (
VVTLibDataSubjectDB,
VVTLibDataCategoryDB,
VVTLibRecipientDB,
VVTLibLegalBasisDB,
VVTLibRetentionRuleDB,
VVTLibTransferMechanismDB,
VVTLibPurposeDB,
VVTLibTomDB,
VVTProcessTemplateDB,
)
from ..db.vvt_models import VVTActivityDB, VVTAuditLogDB
from .tenant_utils import get_tenant_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/vvt", tags=["compliance-vvt-libraries"])
# ============================================================================
# Helper: row → dict
# ============================================================================
def _row_to_dict(row, extra_fields=None):
"""Generic row → dict for library items."""
d = {
"id": row.id,
"label_de": row.label_de,
}
if hasattr(row, 'description_de') and row.description_de:
d["description_de"] = row.description_de
if hasattr(row, 'sort_order'):
d["sort_order"] = row.sort_order
if extra_fields:
for f in extra_fields:
if hasattr(row, f):
val = getattr(row, f)
if val is not None:
d[f] = val
return d
# ============================================================================
# Library Overview
# ============================================================================
@router.get("/libraries")
async def get_libraries_overview(db: Session = Depends(get_db)):
"""Overview of all library types with item counts."""
return {
"libraries": [
{"type": "data-subjects", "count": db.query(VVTLibDataSubjectDB).count()},
{"type": "data-categories", "count": db.query(VVTLibDataCategoryDB).count()},
{"type": "recipients", "count": db.query(VVTLibRecipientDB).count()},
{"type": "legal-bases", "count": db.query(VVTLibLegalBasisDB).count()},
{"type": "retention-rules", "count": db.query(VVTLibRetentionRuleDB).count()},
{"type": "transfer-mechanisms", "count": db.query(VVTLibTransferMechanismDB).count()},
{"type": "purposes", "count": db.query(VVTLibPurposeDB).count()},
{"type": "toms", "count": db.query(VVTLibTomDB).count()},
]
}
# ============================================================================
# Data Subjects
# ============================================================================
@router.get("/libraries/data-subjects")
async def list_data_subjects(
typical_for: Optional[str] = Query(None, description="Filter by business function"),
db: Session = Depends(get_db),
):
query = db.query(VVTLibDataSubjectDB).order_by(VVTLibDataSubjectDB.sort_order)
rows = query.all()
items = [_row_to_dict(r, ["art9_relevant", "typical_for"]) for r in rows]
if typical_for:
items = [i for i in items if typical_for in (i.get("typical_for") or [])]
return items
# ============================================================================
# Data Categories (hierarchical)
# ============================================================================
@router.get("/libraries/data-categories")
async def list_data_categories(
flat: Optional[bool] = Query(False, description="Return flat list instead of tree"),
parent_id: Optional[str] = Query(None),
is_art9: Optional[bool] = Query(None),
db: Session = Depends(get_db),
):
query = db.query(VVTLibDataCategoryDB).order_by(VVTLibDataCategoryDB.sort_order)
if parent_id is not None:
query = query.filter(VVTLibDataCategoryDB.parent_id == parent_id)
if is_art9 is not None:
query = query.filter(VVTLibDataCategoryDB.is_art9 == is_art9)
rows = query.all()
extra = ["parent_id", "is_art9", "is_art10", "risk_weight", "default_retention_rule", "default_legal_basis"]
items = [_row_to_dict(r, extra) for r in rows]
if flat or parent_id is not None or is_art9 is not None:
return items
# Build tree
by_parent: dict = {}
for item in items:
pid = item.get("parent_id")
by_parent.setdefault(pid, []).append(item)
tree = []
for item in by_parent.get(None, []):
children = by_parent.get(item["id"], [])
if children:
item["children"] = children
tree.append(item)
return tree
# ============================================================================
# Recipients
# ============================================================================
@router.get("/libraries/recipients")
async def list_recipients(
type: Optional[str] = Query(None, description="INTERNAL, PROCESSOR, CONTROLLER, AUTHORITY"),
db: Session = Depends(get_db),
):
query = db.query(VVTLibRecipientDB).order_by(VVTLibRecipientDB.sort_order)
if type:
query = query.filter(VVTLibRecipientDB.type == type)
rows = query.all()
return [_row_to_dict(r, ["type", "is_third_country", "country"]) for r in rows]
# ============================================================================
# Legal Bases
# ============================================================================
@router.get("/libraries/legal-bases")
async def list_legal_bases(
is_art9: Optional[bool] = Query(None),
type: Optional[str] = Query(None),
db: Session = Depends(get_db),
):
query = db.query(VVTLibLegalBasisDB).order_by(VVTLibLegalBasisDB.sort_order)
if is_art9 is not None:
query = query.filter(VVTLibLegalBasisDB.is_art9 == is_art9)
if type:
query = query.filter(VVTLibLegalBasisDB.type == type)
rows = query.all()
return [_row_to_dict(r, ["article", "type", "is_art9", "typical_national_law"]) for r in rows]
# ============================================================================
# Retention Rules
# ============================================================================
@router.get("/libraries/retention-rules")
async def list_retention_rules(db: Session = Depends(get_db)):
rows = db.query(VVTLibRetentionRuleDB).order_by(VVTLibRetentionRuleDB.sort_order).all()
return [_row_to_dict(r, ["legal_basis", "duration", "duration_unit", "start_event", "deletion_procedure"]) for r in rows]
# ============================================================================
# Transfer Mechanisms
# ============================================================================
@router.get("/libraries/transfer-mechanisms")
async def list_transfer_mechanisms(db: Session = Depends(get_db)):
rows = db.query(VVTLibTransferMechanismDB).order_by(VVTLibTransferMechanismDB.sort_order).all()
return [_row_to_dict(r, ["article", "requires_tia"]) for r in rows]
# ============================================================================
# Purposes
# ============================================================================
@router.get("/libraries/purposes")
async def list_purposes(
typical_for: Optional[str] = Query(None),
db: Session = Depends(get_db),
):
rows = db.query(VVTLibPurposeDB).order_by(VVTLibPurposeDB.sort_order).all()
items = [_row_to_dict(r, ["typical_legal_basis", "typical_for"]) for r in rows]
if typical_for:
items = [i for i in items if typical_for in (i.get("typical_for") or [])]
return items
# ============================================================================
# TOMs
# ============================================================================
@router.get("/libraries/toms")
async def list_toms(
category: Optional[str] = Query(None),
db: Session = Depends(get_db),
):
query = db.query(VVTLibTomDB).order_by(VVTLibTomDB.sort_order)
if category:
query = query.filter(VVTLibTomDB.category == category)
rows = query.all()
return [_row_to_dict(r, ["category", "art32_reference"]) for r in rows]
# ============================================================================
# Process Templates
# ============================================================================
def _template_to_dict(t: VVTProcessTemplateDB) -> dict:
return {
"id": t.id,
"name": t.name,
"description": t.description,
"business_function": t.business_function,
"purpose_refs": t.purpose_refs or [],
"legal_basis_refs": t.legal_basis_refs or [],
"data_subject_refs": t.data_subject_refs or [],
"data_category_refs": t.data_category_refs or [],
"recipient_refs": t.recipient_refs or [],
"tom_refs": t.tom_refs or [],
"transfer_mechanism_refs": t.transfer_mechanism_refs or [],
"retention_rule_ref": t.retention_rule_ref,
"typical_systems": t.typical_systems or [],
"protection_level": t.protection_level or "MEDIUM",
"dpia_required": t.dpia_required or False,
"risk_score": t.risk_score,
"tags": t.tags or [],
"is_system": t.is_system,
"sort_order": t.sort_order,
}
def _resolve_labels(template_dict: dict, db: Session) -> dict:
"""Resolve library IDs to labels within the template dict."""
resolvers = {
"purpose_refs": (VVTLibPurposeDB, "purpose_labels"),
"legal_basis_refs": (VVTLibLegalBasisDB, "legal_basis_labels"),
"data_subject_refs": (VVTLibDataSubjectDB, "data_subject_labels"),
"data_category_refs": (VVTLibDataCategoryDB, "data_category_labels"),
"recipient_refs": (VVTLibRecipientDB, "recipient_labels"),
"tom_refs": (VVTLibTomDB, "tom_labels"),
"transfer_mechanism_refs": (VVTLibTransferMechanismDB, "transfer_mechanism_labels"),
}
for refs_key, (model, labels_key) in resolvers.items():
ids = template_dict.get(refs_key) or []
if ids:
rows = db.query(model).filter(model.id.in_(ids)).all()
label_map = {r.id: r.label_de for r in rows}
template_dict[labels_key] = {rid: label_map.get(rid, rid) for rid in ids}
# Resolve single retention rule
rr = template_dict.get("retention_rule_ref")
if rr:
row = db.query(VVTLibRetentionRuleDB).filter(VVTLibRetentionRuleDB.id == rr).first()
if row:
template_dict["retention_rule_label"] = row.label_de
return template_dict
@router.get("/templates")
async def list_templates(
business_function: Optional[str] = Query(None),
search: Optional[str] = Query(None),
db: Session = Depends(get_db),
):
"""List process templates (system + tenant)."""
query = db.query(VVTProcessTemplateDB).order_by(VVTProcessTemplateDB.sort_order)
if business_function:
query = query.filter(VVTProcessTemplateDB.business_function == business_function)
if search:
term = f"%{search}%"
query = query.filter(
(VVTProcessTemplateDB.name.ilike(term)) |
(VVTProcessTemplateDB.description.ilike(term))
)
templates = query.all()
return [_template_to_dict(t) for t in templates]
@router.get("/templates/{template_id}")
async def get_template(
template_id: str,
db: Session = Depends(get_db),
):
"""Get a single template with resolved library labels."""
t = db.query(VVTProcessTemplateDB).filter(VVTProcessTemplateDB.id == template_id).first()
if not t:
raise HTTPException(status_code=404, detail=f"Template '{template_id}' not found")
result = _template_to_dict(t)
return _resolve_labels(result, db)
@router.post("/templates/{template_id}/instantiate", status_code=201)
async def instantiate_template(
template_id: str,
http_request: Request,
tid: str = Depends(get_tenant_id),
db: Session = Depends(get_db),
):
"""Create a new VVT activity from a process template."""
t = db.query(VVTProcessTemplateDB).filter(VVTProcessTemplateDB.id == template_id).first()
if not t:
raise HTTPException(status_code=404, detail=f"Template '{template_id}' not found")
# Generate unique VVT-ID
count = db.query(VVTActivityDB).filter(VVTActivityDB.tenant_id == tid).count()
vvt_id = f"VVT-{count + 1:04d}"
# Resolve library IDs to freetext labels for backward-compat fields
purpose_labels = _resolve_ids(db, VVTLibPurposeDB, t.purpose_refs or [])
legal_labels = _resolve_ids(db, VVTLibLegalBasisDB, t.legal_basis_refs or [])
subject_labels = _resolve_ids(db, VVTLibDataSubjectDB, t.data_subject_refs or [])
category_labels = _resolve_ids(db, VVTLibDataCategoryDB, t.data_category_refs or [])
recipient_labels = _resolve_ids(db, VVTLibRecipientDB, t.recipient_refs or [])
# Resolve retention rule
retention_period = {}
if t.retention_rule_ref:
rr = db.query(VVTLibRetentionRuleDB).filter(VVTLibRetentionRuleDB.id == t.retention_rule_ref).first()
if rr:
retention_period = {
"description": rr.label_de,
"legalBasis": rr.legal_basis or "",
"deletionProcedure": rr.deletion_procedure or "",
"duration": rr.duration,
"durationUnit": rr.duration_unit,
}
# Build structured TOMs from tom_refs
structured_toms = {"accessControl": [], "confidentiality": [], "integrity": [], "availability": [], "separation": []}
if t.tom_refs:
tom_rows = db.query(VVTLibTomDB).filter(VVTLibTomDB.id.in_(t.tom_refs)).all()
for tr in tom_rows:
cat = tr.category
if cat in structured_toms:
structured_toms[cat].append(tr.label_de)
act = VVTActivityDB(
tenant_id=tid,
vvt_id=vvt_id,
name=t.name,
description=t.description or "",
purposes=purpose_labels,
legal_bases=[{"type": lid, "description": lbl} for lid, lbl in zip(t.legal_basis_refs or [], legal_labels)],
data_subject_categories=subject_labels,
personal_data_categories=category_labels,
recipient_categories=[{"type": "unknown", "name": lbl} for lbl in recipient_labels],
retention_period=retention_period,
business_function=t.business_function,
systems=[{"systemId": s, "name": s} for s in (t.typical_systems or [])],
protection_level=t.protection_level or "MEDIUM",
dpia_required=t.dpia_required or False,
structured_toms=structured_toms,
status="DRAFT",
created_by=http_request.headers.get("X-User-ID", "system"),
# Library refs
purpose_refs=t.purpose_refs,
legal_basis_refs=t.legal_basis_refs,
data_subject_refs=t.data_subject_refs,
data_category_refs=t.data_category_refs,
recipient_refs=t.recipient_refs,
retention_rule_ref=t.retention_rule_ref,
transfer_mechanism_refs=t.transfer_mechanism_refs,
tom_refs=t.tom_refs,
source_template_id=t.id,
risk_score=t.risk_score,
)
db.add(act)
db.flush()
# Audit log
audit = VVTAuditLogDB(
tenant_id=tid,
action="CREATE",
entity_type="activity",
entity_id=act.id,
changed_by=http_request.headers.get("X-User-ID", "system"),
new_values={"vvt_id": vvt_id, "source_template_id": t.id, "name": t.name},
)
db.add(audit)
db.commit()
db.refresh(act)
# Return full response
from .vvt_routes import _activity_to_response
return _activity_to_response(act)
def _resolve_ids(db: Session, model, ids: list) -> list:
"""Resolve list of library IDs to list of label_de strings."""
if not ids:
return []
rows = db.query(model).filter(model.id.in_(ids)).all()
label_map = {r.id: r.label_de for r in rows}
return [label_map.get(i, i) for i in ids]

View File

@@ -174,6 +174,20 @@ def _activity_to_response(act: VVTActivityDB) -> VVTActivityResponse:
next_review_at=act.next_review_at,
created_by=act.created_by,
dsfa_id=str(act.dsfa_id) if act.dsfa_id else None,
# Library refs
purpose_refs=act.purpose_refs,
legal_basis_refs=act.legal_basis_refs,
data_subject_refs=act.data_subject_refs,
data_category_refs=act.data_category_refs,
recipient_refs=act.recipient_refs,
retention_rule_ref=act.retention_rule_ref,
transfer_mechanism_refs=act.transfer_mechanism_refs,
tom_refs=act.tom_refs,
source_template_id=act.source_template_id,
risk_score=act.risk_score,
linked_loeschfristen_ids=act.linked_loeschfristen_ids,
linked_tom_measure_ids=act.linked_tom_measure_ids,
art30_completeness=act.art30_completeness,
created_at=act.created_at,
updated_at=act.updated_at,
)
@@ -336,6 +350,107 @@ async def delete_activity(
return {"success": True, "message": f"Activity {activity_id} deleted"}
# ============================================================================
# Art. 30 Completeness Check
# ============================================================================
@router.get("/activities/{activity_id}/completeness")
async def get_activity_completeness(
activity_id: str,
tid: str = Depends(get_tenant_id),
db: Session = Depends(get_db),
):
"""Calculate Art. 30 completeness score for a VVT activity."""
act = db.query(VVTActivityDB).filter(
VVTActivityDB.id == activity_id,
VVTActivityDB.tenant_id == tid,
).first()
if not act:
raise HTTPException(status_code=404, detail=f"Activity {activity_id} not found")
return _calculate_completeness(act)
def _calculate_completeness(act: VVTActivityDB) -> dict:
"""Calculate Art. 30 completeness — required fields per DSGVO Art. 30 Abs. 1."""
missing = []
warnings = []
total_checks = 10
passed = 0
# 1. Name/Zweck
if act.name:
passed += 1
else:
missing.append("name")
# 2. Verarbeitungszwecke
has_purposes = bool(act.purposes) or bool(act.purpose_refs)
if has_purposes:
passed += 1
else:
missing.append("purposes")
# 3. Rechtsgrundlage
has_legal = bool(act.legal_bases) or bool(act.legal_basis_refs)
if has_legal:
passed += 1
else:
missing.append("legal_bases")
# 4. Betroffenenkategorien
has_subjects = bool(act.data_subject_categories) or bool(act.data_subject_refs)
if has_subjects:
passed += 1
else:
missing.append("data_subjects")
# 5. Datenkategorien
has_categories = bool(act.personal_data_categories) or bool(act.data_category_refs)
if has_categories:
passed += 1
else:
missing.append("data_categories")
# 6. Empfaenger
has_recipients = bool(act.recipient_categories) or bool(act.recipient_refs)
if has_recipients:
passed += 1
else:
missing.append("recipients")
# 7. Drittland-Uebermittlung (checked but not strictly required)
passed += 1 # always passes — no transfer is valid state
# 8. Loeschfristen
has_retention = bool(act.retention_period and act.retention_period.get('description')) or bool(act.retention_rule_ref)
if has_retention:
passed += 1
else:
missing.append("retention_period")
# 9. TOM-Beschreibung
has_tom = bool(act.tom_description) or bool(act.tom_refs) or bool(act.structured_toms)
if has_tom:
passed += 1
else:
missing.append("tom_description")
# 10. Verantwortlicher
if act.responsible:
passed += 1
else:
missing.append("responsible")
# Warnings
if act.dpia_required and not act.dsfa_id:
warnings.append("dpia_required_but_no_dsfa_linked")
if act.third_country_transfers and not act.transfer_mechanism_refs:
warnings.append("third_country_transfer_without_mechanism")
score = int((passed / total_checks) * 100)
return {"score": score, "missing": missing, "warnings": warnings, "passed": passed, "total": total_checks}
# ============================================================================
# Audit Log
# ============================================================================