feat: Anti-Fake-Evidence System (Phase 1-4b)
Implement full evidence integrity pipeline to prevent compliance theater: - Confidence levels (E0-E4), truth status tracking, assertion engine - Four-Eyes approval workflow, audit trail, reject endpoint - Evidence distribution dashboard, LLM audit routes - Traceability matrix (backend endpoint + Compliance Hub UI tab) - Anti-fake badges, control status machine, normative patterns - 2 migrations, 4 test suites, MkDocs documentation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
@@ -61,6 +61,8 @@ _ROUTER_MODULES = [
|
||||
"evidence_check_routes",
|
||||
"vvt_library_routes",
|
||||
"tom_mapping_routes",
|
||||
"llm_audit_routes",
|
||||
"assertion_routes",
|
||||
]
|
||||
|
||||
_loaded_count = 0
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
"""
|
||||
API routes for Assertion Engine (Anti-Fake-Evidence Phase 2).
|
||||
|
||||
Endpoints:
|
||||
- /assertions: CRUD for assertions
|
||||
- /assertions/extract: Automatic extraction from entity text
|
||||
- /assertions/summary: Stats (total assertions, facts, unverified)
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from classroom_engine.database import get_db
|
||||
|
||||
from ..db.models import AssertionDB
|
||||
from ..services.assertion_engine import extract_assertions
|
||||
from .schemas import (
|
||||
AssertionCreate,
|
||||
AssertionUpdate,
|
||||
AssertionResponse,
|
||||
AssertionListResponse,
|
||||
AssertionSummaryResponse,
|
||||
AssertionExtractRequest,
|
||||
)
|
||||
from .audit_trail_utils import log_audit_trail, generate_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(tags=["compliance-assertions"])
|
||||
|
||||
|
||||
def _build_assertion_response(a: AssertionDB) -> AssertionResponse:
|
||||
return AssertionResponse(
|
||||
id=a.id,
|
||||
tenant_id=a.tenant_id,
|
||||
entity_type=a.entity_type,
|
||||
entity_id=a.entity_id,
|
||||
sentence_text=a.sentence_text,
|
||||
sentence_index=a.sentence_index,
|
||||
assertion_type=a.assertion_type,
|
||||
evidence_ids=a.evidence_ids or [],
|
||||
confidence=a.confidence or 0.0,
|
||||
normative_tier=a.normative_tier,
|
||||
verified_by=a.verified_by,
|
||||
verified_at=a.verified_at,
|
||||
created_at=a.created_at,
|
||||
updated_at=a.updated_at,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/assertions", response_model=AssertionResponse)
|
||||
async def create_assertion(
|
||||
data: AssertionCreate,
|
||||
tenant_id: Optional[str] = Query(None),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Create a single assertion manually."""
|
||||
a = AssertionDB(
|
||||
id=generate_id(),
|
||||
tenant_id=tenant_id,
|
||||
entity_type=data.entity_type,
|
||||
entity_id=data.entity_id,
|
||||
sentence_text=data.sentence_text,
|
||||
assertion_type=data.assertion_type or "assertion",
|
||||
evidence_ids=data.evidence_ids or [],
|
||||
normative_tier=data.normative_tier,
|
||||
)
|
||||
db.add(a)
|
||||
db.commit()
|
||||
db.refresh(a)
|
||||
return _build_assertion_response(a)
|
||||
|
||||
|
||||
@router.get("/assertions", response_model=AssertionListResponse)
|
||||
async def list_assertions(
|
||||
entity_type: Optional[str] = Query(None),
|
||||
entity_id: Optional[str] = Query(None),
|
||||
assertion_type: Optional[str] = Query(None),
|
||||
tenant_id: Optional[str] = Query(None),
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""List assertions with optional filters."""
|
||||
query = db.query(AssertionDB)
|
||||
if entity_type:
|
||||
query = query.filter(AssertionDB.entity_type == entity_type)
|
||||
if entity_id:
|
||||
query = query.filter(AssertionDB.entity_id == entity_id)
|
||||
if assertion_type:
|
||||
query = query.filter(AssertionDB.assertion_type == assertion_type)
|
||||
if tenant_id:
|
||||
query = query.filter(AssertionDB.tenant_id == tenant_id)
|
||||
|
||||
total = query.count()
|
||||
records = query.order_by(AssertionDB.sentence_index.asc()).limit(limit).all()
|
||||
|
||||
return AssertionListResponse(
|
||||
assertions=[_build_assertion_response(a) for a in records],
|
||||
total=total,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/assertions/summary", response_model=AssertionSummaryResponse)
|
||||
async def assertion_summary(
|
||||
tenant_id: Optional[str] = Query(None),
|
||||
entity_type: Optional[str] = Query(None),
|
||||
entity_id: Optional[str] = Query(None),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Summary stats: total assertions, facts, rationale, unverified."""
|
||||
query = db.query(AssertionDB)
|
||||
if tenant_id:
|
||||
query = query.filter(AssertionDB.tenant_id == tenant_id)
|
||||
if entity_type:
|
||||
query = query.filter(AssertionDB.entity_type == entity_type)
|
||||
if entity_id:
|
||||
query = query.filter(AssertionDB.entity_id == entity_id)
|
||||
|
||||
all_records = query.all()
|
||||
|
||||
total = len(all_records)
|
||||
facts = sum(1 for a in all_records if a.assertion_type == "fact")
|
||||
rationale = sum(1 for a in all_records if a.assertion_type == "rationale")
|
||||
unverified = sum(1 for a in all_records if a.assertion_type == "assertion" and not a.verified_by)
|
||||
|
||||
return AssertionSummaryResponse(
|
||||
total_assertions=total,
|
||||
total_facts=facts,
|
||||
total_rationale=rationale,
|
||||
unverified_count=unverified,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/assertions/{assertion_id}", response_model=AssertionResponse)
|
||||
async def get_assertion(
|
||||
assertion_id: str,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Get a single assertion by ID."""
|
||||
a = db.query(AssertionDB).filter(AssertionDB.id == assertion_id).first()
|
||||
if not a:
|
||||
raise HTTPException(status_code=404, detail=f"Assertion {assertion_id} not found")
|
||||
return _build_assertion_response(a)
|
||||
|
||||
|
||||
@router.put("/assertions/{assertion_id}", response_model=AssertionResponse)
|
||||
async def update_assertion(
|
||||
assertion_id: str,
|
||||
data: AssertionUpdate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Update an assertion (e.g. link evidence, change type)."""
|
||||
a = db.query(AssertionDB).filter(AssertionDB.id == assertion_id).first()
|
||||
if not a:
|
||||
raise HTTPException(status_code=404, detail=f"Assertion {assertion_id} not found")
|
||||
|
||||
update_fields = data.model_dump(exclude_unset=True)
|
||||
for key, value in update_fields.items():
|
||||
setattr(a, key, value)
|
||||
a.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
db.refresh(a)
|
||||
return _build_assertion_response(a)
|
||||
|
||||
|
||||
@router.post("/assertions/{assertion_id}/verify", response_model=AssertionResponse)
|
||||
async def verify_assertion(
|
||||
assertion_id: str,
|
||||
verified_by: str = Query(...),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Mark an assertion as verified fact."""
|
||||
a = db.query(AssertionDB).filter(AssertionDB.id == assertion_id).first()
|
||||
if not a:
|
||||
raise HTTPException(status_code=404, detail=f"Assertion {assertion_id} not found")
|
||||
|
||||
a.assertion_type = "fact"
|
||||
a.verified_by = verified_by
|
||||
a.verified_at = datetime.utcnow()
|
||||
a.updated_at = datetime.utcnow()
|
||||
db.commit()
|
||||
db.refresh(a)
|
||||
return _build_assertion_response(a)
|
||||
|
||||
|
||||
@router.post("/assertions/extract", response_model=AssertionListResponse)
|
||||
async def extract_assertions_endpoint(
|
||||
data: AssertionExtractRequest,
|
||||
tenant_id: Optional[str] = Query(None),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Extract assertions from free text and persist them."""
|
||||
extracted = extract_assertions(
|
||||
text=data.text,
|
||||
entity_type=data.entity_type,
|
||||
entity_id=data.entity_id,
|
||||
tenant_id=tenant_id,
|
||||
)
|
||||
|
||||
created = []
|
||||
for item in extracted:
|
||||
a = AssertionDB(
|
||||
id=generate_id(),
|
||||
tenant_id=item["tenant_id"],
|
||||
entity_type=item["entity_type"],
|
||||
entity_id=item["entity_id"],
|
||||
sentence_text=item["sentence_text"],
|
||||
sentence_index=item["sentence_index"],
|
||||
assertion_type=item["assertion_type"],
|
||||
evidence_ids=item["evidence_ids"],
|
||||
normative_tier=item.get("normative_tier"),
|
||||
confidence=item.get("confidence", 0.0),
|
||||
)
|
||||
db.add(a)
|
||||
created.append(a)
|
||||
|
||||
db.commit()
|
||||
for a in created:
|
||||
db.refresh(a)
|
||||
|
||||
return AssertionListResponse(
|
||||
assertions=[_build_assertion_response(a) for a in created],
|
||||
total=len(created),
|
||||
)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Shared audit trail utilities.
|
||||
|
||||
Extracted from isms_routes.py for reuse across evidence, control,
|
||||
and assertion routes.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..db.models import AuditTrailDB
|
||||
|
||||
|
||||
def generate_id() -> str:
|
||||
"""Generate a UUID string."""
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def create_signature(data: str) -> str:
|
||||
"""Create SHA-256 signature."""
|
||||
return hashlib.sha256(data.encode()).hexdigest()
|
||||
|
||||
|
||||
def log_audit_trail(
|
||||
db: Session,
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
entity_name: str,
|
||||
action: str,
|
||||
performed_by: str,
|
||||
field_changed: str = None,
|
||||
old_value: str = None,
|
||||
new_value: str = None,
|
||||
change_summary: str = None,
|
||||
):
|
||||
"""Log an entry to the audit trail."""
|
||||
trail = AuditTrailDB(
|
||||
id=generate_id(),
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
entity_name=entity_name,
|
||||
action=action,
|
||||
field_changed=field_changed,
|
||||
old_value=old_value,
|
||||
new_value=new_value,
|
||||
change_summary=change_summary,
|
||||
performed_by=performed_by,
|
||||
performed_at=datetime.utcnow(),
|
||||
checksum=create_signature(f"{entity_type}|{entity_id}|{action}|{performed_by}"),
|
||||
)
|
||||
db.add(trail)
|
||||
@@ -32,14 +32,21 @@ from ..db import (
|
||||
ControlRepository,
|
||||
EvidenceRepository,
|
||||
RiskRepository,
|
||||
AssertionDB,
|
||||
)
|
||||
from .schemas import (
|
||||
DashboardResponse,
|
||||
MultiDimensionalScore,
|
||||
ExecutiveDashboardResponse,
|
||||
TrendDataPoint,
|
||||
RiskSummary,
|
||||
DeadlineItem,
|
||||
TeamWorkloadItem,
|
||||
TraceabilityAssertion,
|
||||
TraceabilityEvidence,
|
||||
TraceabilityCoverage,
|
||||
TraceabilityControl,
|
||||
TraceabilityMatrixResponse,
|
||||
)
|
||||
from .tenant_utils import get_tenant_id as _get_tenant_id
|
||||
from .db_utils import row_to_dict as _row_to_dict
|
||||
@@ -95,6 +102,14 @@ async def get_dashboard(db: Session = Depends(get_db)):
|
||||
# or compute from by_status dict
|
||||
score = ctrl_stats.get("compliance_score", 0.0)
|
||||
|
||||
# Multi-dimensional score (Anti-Fake-Evidence)
|
||||
try:
|
||||
ms = ctrl_repo.get_multi_dimensional_score()
|
||||
multi_score = MultiDimensionalScore(**ms)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to compute multi-dimensional score: {e}")
|
||||
multi_score = None
|
||||
|
||||
return DashboardResponse(
|
||||
compliance_score=round(score, 1),
|
||||
total_regulations=len(regulations),
|
||||
@@ -107,6 +122,7 @@ async def get_dashboard(db: Session = Depends(get_db)):
|
||||
total_risks=len(risks),
|
||||
risks_by_level=risks_by_level,
|
||||
recent_activity=[],
|
||||
multi_score=multi_score,
|
||||
)
|
||||
|
||||
|
||||
@@ -125,11 +141,18 @@ async def get_compliance_score(db: Session = Depends(get_db)):
|
||||
else:
|
||||
score = 0
|
||||
|
||||
# Multi-dimensional score (Anti-Fake-Evidence)
|
||||
try:
|
||||
multi_score = ctrl_repo.get_multi_dimensional_score()
|
||||
except Exception:
|
||||
multi_score = None
|
||||
|
||||
return {
|
||||
"score": round(score, 1),
|
||||
"total_controls": total,
|
||||
"passing_controls": passing,
|
||||
"partial_controls": partial,
|
||||
"multi_score": multi_score,
|
||||
}
|
||||
|
||||
|
||||
@@ -597,6 +620,158 @@ async def get_score_history(
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Evidence Distribution (Anti-Fake-Evidence Phase 3)
|
||||
# ============================================================================
|
||||
|
||||
@router.get("/dashboard/evidence-distribution")
|
||||
async def get_evidence_distribution(
|
||||
db: Session = Depends(get_db),
|
||||
tenant_id: str = Depends(_get_tenant_id),
|
||||
):
|
||||
"""Evidence counts by confidence level and four-eyes status."""
|
||||
evidence_repo = EvidenceRepository(db)
|
||||
all_evidence = evidence_repo.get_all()
|
||||
|
||||
by_confidence = {"E0": 0, "E1": 0, "E2": 0, "E3": 0, "E4": 0}
|
||||
four_eyes_pending = 0
|
||||
|
||||
for e in all_evidence:
|
||||
level = e.confidence_level.value if e.confidence_level else "E1"
|
||||
if level in by_confidence:
|
||||
by_confidence[level] += 1
|
||||
if e.requires_four_eyes and e.approval_status not in ("approved", "rejected"):
|
||||
four_eyes_pending += 1
|
||||
|
||||
return {
|
||||
"by_confidence": by_confidence,
|
||||
"four_eyes_pending": four_eyes_pending,
|
||||
"total": len(all_evidence),
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Traceability Matrix (Anti-Fake-Evidence Phase 4a)
|
||||
# ============================================================================
|
||||
|
||||
@router.get("/dashboard/traceability-matrix", response_model=TraceabilityMatrixResponse)
|
||||
async def get_traceability_matrix(
|
||||
db: Session = Depends(get_db),
|
||||
tenant_id: str = Depends(_get_tenant_id),
|
||||
):
|
||||
"""
|
||||
Full traceability chain: Control → Evidence → Assertions.
|
||||
|
||||
Loads each entity set once, builds in-memory indices, and nests
|
||||
the result so the frontend can render a matrix view.
|
||||
"""
|
||||
ctrl_repo = ControlRepository(db)
|
||||
evidence_repo = EvidenceRepository(db)
|
||||
|
||||
# 1. Load all three entity sets
|
||||
controls = ctrl_repo.get_all()
|
||||
all_evidence = evidence_repo.get_all()
|
||||
all_assertions = db.query(AssertionDB).filter(
|
||||
AssertionDB.entity_type == "evidence",
|
||||
).all()
|
||||
|
||||
# 2. Index assertions by evidence_id (entity_id)
|
||||
assertions_by_evidence: Dict[str, list] = {}
|
||||
for a in all_assertions:
|
||||
assertions_by_evidence.setdefault(a.entity_id, []).append(a)
|
||||
|
||||
# 3. Index evidence by control_id
|
||||
evidence_by_control: Dict[str, list] = {}
|
||||
for e in all_evidence:
|
||||
evidence_by_control.setdefault(str(e.control_id), []).append(e)
|
||||
|
||||
# 4. Build nested response
|
||||
result_controls: list = []
|
||||
total_controls = 0
|
||||
covered_controls = 0
|
||||
fully_verified = 0
|
||||
|
||||
for ctrl in controls:
|
||||
total_controls += 1
|
||||
ctrl_id = str(ctrl.id)
|
||||
ctrl_evidence = evidence_by_control.get(ctrl_id, [])
|
||||
|
||||
nested_evidence: list = []
|
||||
has_evidence = len(ctrl_evidence) > 0
|
||||
has_assertions = False
|
||||
all_verified = True
|
||||
min_conf: Optional[str] = None
|
||||
conf_order = {"E0": 0, "E1": 1, "E2": 2, "E3": 3, "E4": 4}
|
||||
|
||||
for e in ctrl_evidence:
|
||||
ev_id = str(e.id)
|
||||
ev_assertions = assertions_by_evidence.get(ev_id, [])
|
||||
|
||||
nested_assertions = [
|
||||
TraceabilityAssertion(
|
||||
id=str(a.id),
|
||||
sentence_text=a.sentence_text,
|
||||
assertion_type=a.assertion_type or "assertion",
|
||||
confidence=a.confidence or 0.0,
|
||||
verified=a.verified_by is not None,
|
||||
)
|
||||
for a in ev_assertions
|
||||
]
|
||||
|
||||
if nested_assertions:
|
||||
has_assertions = True
|
||||
for na in nested_assertions:
|
||||
if not na.verified:
|
||||
all_verified = False
|
||||
|
||||
conf = e.confidence_level.value if e.confidence_level else "E1"
|
||||
if min_conf is None or conf_order.get(conf, 1) < conf_order.get(min_conf, 1):
|
||||
min_conf = conf
|
||||
|
||||
nested_evidence.append(TraceabilityEvidence(
|
||||
id=ev_id,
|
||||
title=e.title,
|
||||
evidence_type=e.evidence_type,
|
||||
confidence_level=conf,
|
||||
status=e.status.value if e.status else "valid",
|
||||
assertions=nested_assertions,
|
||||
))
|
||||
|
||||
if not has_assertions:
|
||||
all_verified = False
|
||||
|
||||
if has_evidence:
|
||||
covered_controls += 1
|
||||
if has_evidence and has_assertions and all_verified:
|
||||
fully_verified += 1
|
||||
|
||||
coverage = TraceabilityCoverage(
|
||||
has_evidence=has_evidence,
|
||||
has_assertions=has_assertions,
|
||||
all_assertions_verified=all_verified,
|
||||
min_confidence_level=min_conf,
|
||||
)
|
||||
|
||||
result_controls.append(TraceabilityControl(
|
||||
id=ctrl_id,
|
||||
control_id=ctrl.control_id,
|
||||
title=ctrl.title,
|
||||
status=ctrl.status.value if ctrl.status else "planned",
|
||||
domain=ctrl.domain.value if ctrl.domain else "unknown",
|
||||
evidence=nested_evidence,
|
||||
coverage=coverage,
|
||||
))
|
||||
|
||||
summary = {
|
||||
"total_controls": total_controls,
|
||||
"covered_controls": covered_controls,
|
||||
"fully_verified": fully_verified,
|
||||
"uncovered_controls": total_controls - covered_controls,
|
||||
}
|
||||
|
||||
return TraceabilityMatrixResponse(controls=result_controls, summary=summary)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Reports
|
||||
# ============================================================================
|
||||
|
||||
@@ -26,17 +26,102 @@ from ..db import (
|
||||
ControlRepository,
|
||||
EvidenceRepository,
|
||||
EvidenceStatusEnum,
|
||||
EvidenceConfidenceEnum,
|
||||
EvidenceTruthStatusEnum,
|
||||
)
|
||||
from ..db.models import EvidenceDB, ControlDB
|
||||
from ..db.models import EvidenceDB, ControlDB, AuditTrailDB
|
||||
from ..services.auto_risk_updater import AutoRiskUpdater
|
||||
from .schemas import (
|
||||
EvidenceCreate, EvidenceResponse, EvidenceListResponse,
|
||||
EvidenceRejectRequest,
|
||||
)
|
||||
from .audit_trail_utils import log_audit_trail
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(tags=["compliance-evidence"])
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Anti-Fake-Evidence: Four-Eyes Domain Check
|
||||
# ============================================================================
|
||||
|
||||
FOUR_EYES_DOMAINS = {"gov", "priv"}
|
||||
|
||||
|
||||
def _requires_four_eyes(control_domain: str) -> bool:
|
||||
"""Controls in governance/privacy domains require two independent reviewers."""
|
||||
return control_domain in FOUR_EYES_DOMAINS
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Anti-Fake-Evidence: Auto-Classification Helpers
|
||||
# ============================================================================
|
||||
|
||||
def _classify_confidence(source: Optional[str], evidence_type: Optional[str] = None, artifact_hash: Optional[str] = None) -> EvidenceConfidenceEnum:
|
||||
"""Classify evidence confidence level based on source and metadata."""
|
||||
if source == "ci_pipeline":
|
||||
return EvidenceConfidenceEnum.E3
|
||||
if source == "api" and artifact_hash:
|
||||
return EvidenceConfidenceEnum.E3
|
||||
if source == "api":
|
||||
return EvidenceConfidenceEnum.E3
|
||||
if source in ("manual", "upload"):
|
||||
return EvidenceConfidenceEnum.E1
|
||||
if source == "generated":
|
||||
return EvidenceConfidenceEnum.E0
|
||||
# Default for unknown sources
|
||||
return EvidenceConfidenceEnum.E1
|
||||
|
||||
|
||||
def _classify_truth_status(source: Optional[str]) -> EvidenceTruthStatusEnum:
|
||||
"""Classify evidence truth status based on source."""
|
||||
if source == "ci_pipeline":
|
||||
return EvidenceTruthStatusEnum.OBSERVED
|
||||
if source in ("manual", "upload"):
|
||||
return EvidenceTruthStatusEnum.UPLOADED
|
||||
if source == "generated":
|
||||
return EvidenceTruthStatusEnum.GENERATED
|
||||
if source == "api":
|
||||
return EvidenceTruthStatusEnum.OBSERVED
|
||||
return EvidenceTruthStatusEnum.UPLOADED
|
||||
|
||||
|
||||
def _build_evidence_response(e: EvidenceDB) -> EvidenceResponse:
|
||||
"""Build an EvidenceResponse from an EvidenceDB, including anti-fake fields."""
|
||||
return EvidenceResponse(
|
||||
id=e.id,
|
||||
control_id=e.control_id,
|
||||
evidence_type=e.evidence_type,
|
||||
title=e.title,
|
||||
description=e.description,
|
||||
artifact_path=e.artifact_path,
|
||||
artifact_url=e.artifact_url,
|
||||
artifact_hash=e.artifact_hash,
|
||||
file_size_bytes=e.file_size_bytes,
|
||||
mime_type=e.mime_type,
|
||||
valid_from=e.valid_from,
|
||||
valid_until=e.valid_until,
|
||||
status=e.status.value if e.status else None,
|
||||
source=e.source,
|
||||
ci_job_id=e.ci_job_id,
|
||||
uploaded_by=e.uploaded_by,
|
||||
collected_at=e.collected_at,
|
||||
created_at=e.created_at,
|
||||
confidence_level=e.confidence_level.value if e.confidence_level else None,
|
||||
truth_status=e.truth_status.value if e.truth_status else None,
|
||||
generation_mode=e.generation_mode,
|
||||
may_be_used_as_evidence=e.may_be_used_as_evidence,
|
||||
reviewed_by=e.reviewed_by,
|
||||
reviewed_at=e.reviewed_at,
|
||||
approval_status=e.approval_status,
|
||||
first_reviewer=e.first_reviewer,
|
||||
first_reviewed_at=e.first_reviewed_at,
|
||||
second_reviewer=e.second_reviewer,
|
||||
second_reviewed_at=e.second_reviewed_at,
|
||||
requires_four_eyes=e.requires_four_eyes,
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Evidence
|
||||
# ============================================================================
|
||||
@@ -80,29 +165,7 @@ async def list_evidence(
|
||||
offset = (page - 1) * limit
|
||||
evidence = evidence[offset:offset + limit]
|
||||
|
||||
results = [
|
||||
EvidenceResponse(
|
||||
id=e.id,
|
||||
control_id=e.control_id,
|
||||
evidence_type=e.evidence_type,
|
||||
title=e.title,
|
||||
description=e.description,
|
||||
artifact_path=e.artifact_path,
|
||||
artifact_url=e.artifact_url,
|
||||
artifact_hash=e.artifact_hash,
|
||||
file_size_bytes=e.file_size_bytes,
|
||||
mime_type=e.mime_type,
|
||||
valid_from=e.valid_from,
|
||||
valid_until=e.valid_until,
|
||||
status=e.status.value if e.status else None,
|
||||
source=e.source,
|
||||
ci_job_id=e.ci_job_id,
|
||||
uploaded_by=e.uploaded_by,
|
||||
collected_at=e.collected_at,
|
||||
created_at=e.created_at,
|
||||
)
|
||||
for e in evidence
|
||||
]
|
||||
results = [_build_evidence_response(e) for e in evidence]
|
||||
|
||||
return EvidenceListResponse(evidence=results, total=total)
|
||||
|
||||
@@ -121,6 +184,22 @@ async def create_evidence(
|
||||
if not control:
|
||||
raise HTTPException(status_code=404, detail=f"Control {evidence_data.control_id} not found")
|
||||
|
||||
source = evidence_data.source or "api"
|
||||
confidence = _classify_confidence(source, evidence_data.evidence_type)
|
||||
truth = _classify_truth_status(source)
|
||||
|
||||
# Allow explicit override from request
|
||||
if evidence_data.confidence_level:
|
||||
try:
|
||||
confidence = EvidenceConfidenceEnum(evidence_data.confidence_level)
|
||||
except ValueError:
|
||||
pass
|
||||
if evidence_data.truth_status:
|
||||
try:
|
||||
truth = EvidenceTruthStatusEnum(evidence_data.truth_status)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
evidence = repo.create(
|
||||
control_id=control.id,
|
||||
evidence_type=evidence_data.evidence_type,
|
||||
@@ -129,31 +208,34 @@ async def create_evidence(
|
||||
artifact_url=evidence_data.artifact_url,
|
||||
valid_from=evidence_data.valid_from,
|
||||
valid_until=evidence_data.valid_until,
|
||||
source=evidence_data.source or "api",
|
||||
source=source,
|
||||
ci_job_id=evidence_data.ci_job_id,
|
||||
)
|
||||
|
||||
# Set anti-fake-evidence fields
|
||||
evidence.confidence_level = confidence
|
||||
evidence.truth_status = truth
|
||||
# Generated evidence should not be used as evidence by default
|
||||
if truth == EvidenceTruthStatusEnum.GENERATED:
|
||||
evidence.may_be_used_as_evidence = False
|
||||
|
||||
# Four-Eyes: check if the linked control's domain requires it
|
||||
control_domain = control.domain.value if control.domain else ""
|
||||
if _requires_four_eyes(control_domain):
|
||||
evidence.requires_four_eyes = True
|
||||
evidence.approval_status = "pending_first"
|
||||
|
||||
db.commit()
|
||||
|
||||
# Audit trail
|
||||
log_audit_trail(
|
||||
db, "evidence", evidence.id, evidence.title, "create",
|
||||
performed_by=evidence_data.source or "api",
|
||||
change_summary=f"Evidence created with confidence={confidence.value}, truth={truth.value}",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
return EvidenceResponse(
|
||||
id=evidence.id,
|
||||
control_id=evidence.control_id,
|
||||
evidence_type=evidence.evidence_type,
|
||||
title=evidence.title,
|
||||
description=evidence.description,
|
||||
artifact_path=evidence.artifact_path,
|
||||
artifact_url=evidence.artifact_url,
|
||||
artifact_hash=evidence.artifact_hash,
|
||||
file_size_bytes=evidence.file_size_bytes,
|
||||
mime_type=evidence.mime_type,
|
||||
valid_from=evidence.valid_from,
|
||||
valid_until=evidence.valid_until,
|
||||
status=evidence.status.value if evidence.status else None,
|
||||
source=evidence.source,
|
||||
ci_job_id=evidence.ci_job_id,
|
||||
uploaded_by=evidence.uploaded_by,
|
||||
collected_at=evidence.collected_at,
|
||||
created_at=evidence.created_at,
|
||||
)
|
||||
return _build_evidence_response(evidence)
|
||||
|
||||
|
||||
@router.delete("/evidence/{evidence_id}")
|
||||
@@ -223,28 +305,20 @@ async def upload_evidence(
|
||||
mime_type=file.content_type,
|
||||
source="upload",
|
||||
)
|
||||
|
||||
# Upload evidence → E1 + uploaded
|
||||
evidence.confidence_level = EvidenceConfidenceEnum.E1
|
||||
evidence.truth_status = EvidenceTruthStatusEnum.UPLOADED
|
||||
|
||||
# Four-Eyes: check if the linked control's domain requires it
|
||||
control_domain = control.domain.value if control.domain else ""
|
||||
if _requires_four_eyes(control_domain):
|
||||
evidence.requires_four_eyes = True
|
||||
evidence.approval_status = "pending_first"
|
||||
|
||||
db.commit()
|
||||
|
||||
return EvidenceResponse(
|
||||
id=evidence.id,
|
||||
control_id=evidence.control_id,
|
||||
evidence_type=evidence.evidence_type,
|
||||
title=evidence.title,
|
||||
description=evidence.description,
|
||||
artifact_path=evidence.artifact_path,
|
||||
artifact_url=evidence.artifact_url,
|
||||
artifact_hash=evidence.artifact_hash,
|
||||
file_size_bytes=evidence.file_size_bytes,
|
||||
mime_type=evidence.mime_type,
|
||||
valid_from=evidence.valid_from,
|
||||
valid_until=evidence.valid_until,
|
||||
status=evidence.status.value if evidence.status else None,
|
||||
source=evidence.source,
|
||||
ci_job_id=evidence.ci_job_id,
|
||||
uploaded_by=evidence.uploaded_by,
|
||||
collected_at=evidence.collected_at,
|
||||
created_at=evidence.created_at,
|
||||
)
|
||||
return _build_evidence_response(evidence)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
@@ -357,7 +431,7 @@ def _store_evidence(
|
||||
with open(file_path, "w") as f:
|
||||
json.dump(report_data or {}, f, indent=2)
|
||||
|
||||
# Create evidence record
|
||||
# Create evidence record with anti-fake-evidence classification
|
||||
evidence = EvidenceDB(
|
||||
id=str(uuid_module.uuid4()),
|
||||
control_id=control_db_id,
|
||||
@@ -373,6 +447,10 @@ def _store_evidence(
|
||||
valid_from=datetime.utcnow(),
|
||||
valid_until=datetime.utcnow() + timedelta(days=90),
|
||||
status=EvidenceStatusEnum(parsed["evidence_status"]),
|
||||
# CI pipeline evidence → E3 observed (system-observed, hash-verified)
|
||||
confidence_level=EvidenceConfidenceEnum.E3,
|
||||
truth_status=EvidenceTruthStatusEnum.OBSERVED,
|
||||
may_be_used_as_evidence=True,
|
||||
)
|
||||
db.add(evidence)
|
||||
db.commit()
|
||||
@@ -639,3 +717,169 @@ async def get_ci_evidence_status(
|
||||
"total_evidence": len(evidence_list),
|
||||
"controls": result,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Evidence Review (Anti-Fake-Evidence)
|
||||
# ============================================================================
|
||||
|
||||
from pydantic import BaseModel as _BaseModel
|
||||
|
||||
class _EvidenceReviewRequest(_BaseModel):
|
||||
confidence_level: Optional[str] = None
|
||||
truth_status: Optional[str] = None
|
||||
reviewed_by: str
|
||||
|
||||
|
||||
@router.patch("/evidence/{evidence_id}/review", response_model=EvidenceResponse)
|
||||
async def review_evidence(
|
||||
evidence_id: str,
|
||||
review: _EvidenceReviewRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Review evidence: upgrade confidence level and/or change truth status.
|
||||
|
||||
For Four-Eyes evidence, the first reviewer sets first_reviewer and
|
||||
approval_status='first_approved'. A second (different) reviewer then
|
||||
sets second_reviewer and approval_status='approved'.
|
||||
"""
|
||||
evidence = db.query(EvidenceDB).filter(EvidenceDB.id == evidence_id).first()
|
||||
if not evidence:
|
||||
raise HTTPException(status_code=404, detail=f"Evidence {evidence_id} not found")
|
||||
|
||||
old_confidence = evidence.confidence_level.value if evidence.confidence_level else None
|
||||
old_truth = evidence.truth_status.value if evidence.truth_status else None
|
||||
|
||||
if review.confidence_level:
|
||||
try:
|
||||
evidence.confidence_level = EvidenceConfidenceEnum(review.confidence_level)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid confidence_level: {review.confidence_level}")
|
||||
|
||||
if review.truth_status:
|
||||
try:
|
||||
evidence.truth_status = EvidenceTruthStatusEnum(review.truth_status)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid truth_status: {review.truth_status}")
|
||||
|
||||
# Four-Eyes branching
|
||||
if evidence.requires_four_eyes:
|
||||
status = evidence.approval_status or "none"
|
||||
if status in ("none", "pending_first"):
|
||||
evidence.first_reviewer = review.reviewed_by
|
||||
evidence.first_reviewed_at = datetime.utcnow()
|
||||
evidence.approval_status = "first_approved"
|
||||
elif status == "first_approved":
|
||||
if review.reviewed_by == evidence.first_reviewer:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Four-Eyes: second reviewer must be different from first reviewer",
|
||||
)
|
||||
evidence.second_reviewer = review.reviewed_by
|
||||
evidence.second_reviewed_at = datetime.utcnow()
|
||||
evidence.approval_status = "approved"
|
||||
elif status == "approved":
|
||||
raise HTTPException(status_code=400, detail="Evidence already approved")
|
||||
elif status == "rejected":
|
||||
raise HTTPException(status_code=400, detail="Evidence was rejected — create new evidence instead")
|
||||
|
||||
evidence.reviewed_by = review.reviewed_by
|
||||
evidence.reviewed_at = datetime.utcnow()
|
||||
db.commit()
|
||||
|
||||
# Audit trail
|
||||
new_confidence = evidence.confidence_level.value if evidence.confidence_level else None
|
||||
if old_confidence != new_confidence:
|
||||
log_audit_trail(
|
||||
db, "evidence", evidence_id, evidence.title, "review",
|
||||
performed_by=review.reviewed_by,
|
||||
field_changed="confidence_level",
|
||||
old_value=old_confidence,
|
||||
new_value=new_confidence,
|
||||
)
|
||||
new_truth = evidence.truth_status.value if evidence.truth_status else None
|
||||
if old_truth != new_truth:
|
||||
log_audit_trail(
|
||||
db, "evidence", evidence_id, evidence.title, "review",
|
||||
performed_by=review.reviewed_by,
|
||||
field_changed="truth_status",
|
||||
old_value=old_truth,
|
||||
new_value=new_truth,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
db.refresh(evidence)
|
||||
return _build_evidence_response(evidence)
|
||||
|
||||
|
||||
@router.patch("/evidence/{evidence_id}/reject", response_model=EvidenceResponse)
|
||||
async def reject_evidence(
|
||||
evidence_id: str,
|
||||
body: EvidenceRejectRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Reject evidence (sets approval_status='rejected')."""
|
||||
evidence = db.query(EvidenceDB).filter(EvidenceDB.id == evidence_id).first()
|
||||
if not evidence:
|
||||
raise HTTPException(status_code=404, detail=f"Evidence {evidence_id} not found")
|
||||
|
||||
evidence.approval_status = "rejected"
|
||||
evidence.reviewed_by = body.reviewed_by
|
||||
evidence.reviewed_at = datetime.utcnow()
|
||||
db.commit()
|
||||
|
||||
log_audit_trail(
|
||||
db, "evidence", evidence_id, evidence.title, "reject",
|
||||
performed_by=body.reviewed_by,
|
||||
change_summary=body.rejection_reason or "Evidence rejected",
|
||||
)
|
||||
db.commit()
|
||||
|
||||
db.refresh(evidence)
|
||||
return _build_evidence_response(evidence)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Audit Trail Query
|
||||
# ============================================================================
|
||||
|
||||
@router.get("/audit-trail")
|
||||
async def get_audit_trail(
|
||||
entity_type: Optional[str] = Query(None),
|
||||
entity_id: Optional[str] = Query(None),
|
||||
action: Optional[str] = Query(None),
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Query audit trail entries for an entity."""
|
||||
query = db.query(AuditTrailDB)
|
||||
if entity_type:
|
||||
query = query.filter(AuditTrailDB.entity_type == entity_type)
|
||||
if entity_id:
|
||||
query = query.filter(AuditTrailDB.entity_id == entity_id)
|
||||
if action:
|
||||
query = query.filter(AuditTrailDB.action == action)
|
||||
|
||||
records = query.order_by(AuditTrailDB.performed_at.desc()).limit(limit).all()
|
||||
|
||||
return {
|
||||
"entries": [
|
||||
{
|
||||
"id": r.id,
|
||||
"entity_type": r.entity_type,
|
||||
"entity_id": r.entity_id,
|
||||
"entity_name": r.entity_name,
|
||||
"action": r.action,
|
||||
"field_changed": r.field_changed,
|
||||
"old_value": r.old_value,
|
||||
"new_value": r.new_value,
|
||||
"change_summary": r.change_summary,
|
||||
"performed_by": r.performed_by,
|
||||
"performed_at": r.performed_at.isoformat() if r.performed_at else None,
|
||||
"checksum": r.checksum,
|
||||
}
|
||||
for r in records
|
||||
],
|
||||
"total": len(records),
|
||||
}
|
||||
|
||||
@@ -73,39 +73,8 @@ def generate_id() -> str:
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def create_signature(data: str) -> str:
|
||||
"""Create SHA-256 signature."""
|
||||
return hashlib.sha256(data.encode()).hexdigest()
|
||||
|
||||
|
||||
def log_audit_trail(
|
||||
db: Session,
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
entity_name: str,
|
||||
action: str,
|
||||
performed_by: str,
|
||||
field_changed: str = None,
|
||||
old_value: str = None,
|
||||
new_value: str = None,
|
||||
change_summary: str = None
|
||||
):
|
||||
"""Log an entry to the audit trail."""
|
||||
trail = AuditTrailDB(
|
||||
id=generate_id(),
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
entity_name=entity_name,
|
||||
action=action,
|
||||
field_changed=field_changed,
|
||||
old_value=old_value,
|
||||
new_value=new_value,
|
||||
change_summary=change_summary,
|
||||
performed_by=performed_by,
|
||||
performed_at=datetime.utcnow(),
|
||||
checksum=create_signature(f"{entity_type}|{entity_id}|{action}|{performed_by}")
|
||||
)
|
||||
db.add(trail)
|
||||
# Shared audit trail utilities — canonical implementation in audit_trail_utils.py
|
||||
from .audit_trail_utils import log_audit_trail, create_signature # noqa: E402
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
"""
|
||||
FastAPI routes for LLM Generation Audit Trail.
|
||||
|
||||
Endpoints:
|
||||
- POST /llm-audit: Record an LLM generation event
|
||||
- GET /llm-audit: List audit records with filters
|
||||
"""
|
||||
|
||||
import logging
|
||||
import uuid as uuid_module
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from classroom_engine.database import get_db
|
||||
from ..db.models import LLMGenerationAuditDB
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(tags=["compliance-llm-audit"])
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Schemas
|
||||
# ============================================================================
|
||||
|
||||
class LLMAuditCreate(BaseModel):
|
||||
entity_type: str
|
||||
entity_id: Optional[str] = None
|
||||
generation_mode: str
|
||||
truth_status: str = "generated"
|
||||
may_be_used_as_evidence: bool = False
|
||||
llm_model: Optional[str] = None
|
||||
llm_provider: Optional[str] = None
|
||||
prompt_hash: Optional[str] = None
|
||||
input_summary: Optional[str] = None
|
||||
output_summary: Optional[str] = None
|
||||
metadata: Optional[dict] = None
|
||||
tenant_id: Optional[str] = None
|
||||
|
||||
|
||||
class LLMAuditResponse(BaseModel):
|
||||
id: str
|
||||
tenant_id: Optional[str] = None
|
||||
entity_type: str
|
||||
entity_id: Optional[str] = None
|
||||
generation_mode: str
|
||||
truth_status: str
|
||||
may_be_used_as_evidence: bool
|
||||
llm_model: Optional[str] = None
|
||||
llm_provider: Optional[str] = None
|
||||
prompt_hash: Optional[str] = None
|
||||
input_summary: Optional[str] = None
|
||||
output_summary: Optional[str] = None
|
||||
metadata: Optional[dict] = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Routes
|
||||
# ============================================================================
|
||||
|
||||
@router.post("/llm-audit", response_model=LLMAuditResponse)
|
||||
async def create_llm_audit(
|
||||
data: LLMAuditCreate,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""Record an LLM generation event for audit trail."""
|
||||
from ..db.models import EvidenceTruthStatusEnum
|
||||
|
||||
# Validate truth_status
|
||||
try:
|
||||
truth_enum = EvidenceTruthStatusEnum(data.truth_status)
|
||||
except ValueError:
|
||||
truth_enum = EvidenceTruthStatusEnum.GENERATED
|
||||
|
||||
record = LLMGenerationAuditDB(
|
||||
id=str(uuid_module.uuid4()),
|
||||
tenant_id=data.tenant_id,
|
||||
entity_type=data.entity_type,
|
||||
entity_id=data.entity_id,
|
||||
generation_mode=data.generation_mode,
|
||||
truth_status=truth_enum,
|
||||
may_be_used_as_evidence=data.may_be_used_as_evidence,
|
||||
llm_model=data.llm_model,
|
||||
llm_provider=data.llm_provider,
|
||||
prompt_hash=data.prompt_hash,
|
||||
input_summary=data.input_summary[:500] if data.input_summary else None,
|
||||
output_summary=data.output_summary[:500] if data.output_summary else None,
|
||||
extra_metadata=data.metadata or {},
|
||||
)
|
||||
db.add(record)
|
||||
db.commit()
|
||||
db.refresh(record)
|
||||
|
||||
return LLMAuditResponse(
|
||||
id=record.id,
|
||||
tenant_id=record.tenant_id,
|
||||
entity_type=record.entity_type,
|
||||
entity_id=record.entity_id,
|
||||
generation_mode=record.generation_mode,
|
||||
truth_status=record.truth_status.value if record.truth_status else "generated",
|
||||
may_be_used_as_evidence=record.may_be_used_as_evidence,
|
||||
llm_model=record.llm_model,
|
||||
llm_provider=record.llm_provider,
|
||||
prompt_hash=record.prompt_hash,
|
||||
input_summary=record.input_summary,
|
||||
output_summary=record.output_summary,
|
||||
metadata=record.extra_metadata,
|
||||
created_at=record.created_at,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/llm-audit")
|
||||
async def list_llm_audit(
|
||||
entity_type: Optional[str] = Query(None),
|
||||
entity_id: Optional[str] = Query(None),
|
||||
page: int = Query(1, ge=1),
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""List LLM generation audit records with optional filters."""
|
||||
query = db.query(LLMGenerationAuditDB)
|
||||
|
||||
if entity_type:
|
||||
query = query.filter(LLMGenerationAuditDB.entity_type == entity_type)
|
||||
if entity_id:
|
||||
query = query.filter(LLMGenerationAuditDB.entity_id == entity_id)
|
||||
|
||||
total = query.count()
|
||||
offset = (page - 1) * limit
|
||||
records = query.order_by(LLMGenerationAuditDB.created_at.desc()).offset(offset).limit(limit).all()
|
||||
|
||||
return {
|
||||
"records": [
|
||||
LLMAuditResponse(
|
||||
id=r.id,
|
||||
tenant_id=r.tenant_id,
|
||||
entity_type=r.entity_type,
|
||||
entity_id=r.entity_id,
|
||||
generation_mode=r.generation_mode,
|
||||
truth_status=r.truth_status.value if r.truth_status else "generated",
|
||||
may_be_used_as_evidence=r.may_be_used_as_evidence,
|
||||
llm_model=r.llm_model,
|
||||
llm_provider=r.llm_provider,
|
||||
prompt_hash=r.prompt_hash,
|
||||
input_summary=r.input_summary,
|
||||
output_summary=r.output_summary,
|
||||
metadata=r.extra_metadata,
|
||||
created_at=r.created_at,
|
||||
)
|
||||
for r in records
|
||||
],
|
||||
"total": total,
|
||||
"page": page,
|
||||
"limit": limit,
|
||||
}
|
||||
@@ -25,6 +25,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from classroom_engine.database import get_db
|
||||
|
||||
from .audit_trail_utils import log_audit_trail
|
||||
from ..db import (
|
||||
RegulationRepository,
|
||||
RequirementRepository,
|
||||
@@ -595,6 +596,7 @@ async def get_control(control_id: str, db: Session = Depends(get_db)):
|
||||
review_frequency_days=control.review_frequency_days,
|
||||
status=control.status.value if control.status else None,
|
||||
status_notes=control.status_notes,
|
||||
status_justification=control.status_justification,
|
||||
last_reviewed_at=control.last_reviewed_at,
|
||||
next_review_at=control.next_review_at,
|
||||
created_at=control.created_at,
|
||||
@@ -617,16 +619,52 @@ async def update_control(
|
||||
|
||||
update_data = update.model_dump(exclude_unset=True)
|
||||
|
||||
# Convert status string to enum
|
||||
# Convert status string to enum and validate transition
|
||||
if "status" in update_data:
|
||||
try:
|
||||
update_data["status"] = ControlStatusEnum(update_data["status"])
|
||||
new_status_enum = ControlStatusEnum(update_data["status"])
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid status: {update_data['status']}")
|
||||
|
||||
# Validate status transition (Anti-Fake-Evidence)
|
||||
from ..services.control_status_machine import validate_transition
|
||||
current_status = control.status.value if control.status else "planned"
|
||||
evidence_list = db.query(EvidenceDB).filter(EvidenceDB.control_id == control.id).all()
|
||||
allowed, violations = validate_transition(
|
||||
current_status=current_status,
|
||||
new_status=update_data["status"],
|
||||
evidence_list=evidence_list,
|
||||
status_justification=update_data.get("status_justification") or update_data.get("status_notes"),
|
||||
)
|
||||
if not allowed:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"error": "Status transition not allowed",
|
||||
"current_status": current_status,
|
||||
"requested_status": update_data["status"],
|
||||
"violations": violations,
|
||||
}
|
||||
)
|
||||
|
||||
update_data["status"] = new_status_enum
|
||||
|
||||
updated = repo.update(control.id, **update_data)
|
||||
db.commit()
|
||||
|
||||
# Audit trail for status changes
|
||||
new_status = updated.status.value if updated.status else None
|
||||
if "status" in update.model_dump(exclude_unset=True) and current_status != new_status:
|
||||
log_audit_trail(
|
||||
db, "control", control.id, updated.control_id or updated.title,
|
||||
"status_change",
|
||||
performed_by=update.owner or "system",
|
||||
field_changed="status",
|
||||
old_value=current_status,
|
||||
new_value=new_status,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
return ControlResponse(
|
||||
id=updated.id,
|
||||
control_id=updated.control_id,
|
||||
@@ -645,6 +683,7 @@ async def update_control(
|
||||
review_frequency_days=updated.review_frequency_days,
|
||||
status=updated.status.value if updated.status else None,
|
||||
status_notes=updated.status_notes,
|
||||
status_justification=updated.status_justification,
|
||||
last_reviewed_at=updated.last_reviewed_at,
|
||||
next_review_at=updated.next_review_at,
|
||||
created_at=updated.created_at,
|
||||
@@ -690,6 +729,7 @@ async def review_control(
|
||||
review_frequency_days=updated.review_frequency_days,
|
||||
status=updated.status.value if updated.status else None,
|
||||
status_notes=updated.status_notes,
|
||||
status_justification=updated.status_justification,
|
||||
last_reviewed_at=updated.last_reviewed_at,
|
||||
next_review_at=updated.next_review_at,
|
||||
created_at=updated.created_at,
|
||||
|
||||
@@ -43,6 +43,7 @@ class ControlStatus(str):
|
||||
FAIL = "fail"
|
||||
NOT_APPLICABLE = "n/a"
|
||||
PLANNED = "planned"
|
||||
IN_PROGRESS = "in_progress"
|
||||
|
||||
|
||||
class RiskLevel(str):
|
||||
@@ -209,12 +210,14 @@ class ControlUpdate(BaseModel):
|
||||
owner: Optional[str] = None
|
||||
status: Optional[str] = None
|
||||
status_notes: Optional[str] = None
|
||||
status_justification: Optional[str] = None
|
||||
|
||||
|
||||
class ControlResponse(ControlBase):
|
||||
id: str
|
||||
status: str
|
||||
status_notes: Optional[str] = None
|
||||
status_justification: Optional[str] = None
|
||||
last_reviewed_at: Optional[datetime] = None
|
||||
next_review_at: Optional[datetime] = None
|
||||
created_at: datetime
|
||||
@@ -291,7 +294,8 @@ class EvidenceBase(BaseModel):
|
||||
|
||||
|
||||
class EvidenceCreate(EvidenceBase):
|
||||
pass
|
||||
confidence_level: Optional[str] = None
|
||||
truth_status: Optional[str] = None
|
||||
|
||||
|
||||
class EvidenceResponse(EvidenceBase):
|
||||
@@ -304,6 +308,20 @@ class EvidenceResponse(EvidenceBase):
|
||||
uploaded_by: Optional[str] = None
|
||||
collected_at: datetime
|
||||
created_at: datetime
|
||||
# Anti-Fake-Evidence fields
|
||||
confidence_level: Optional[str] = None
|
||||
truth_status: Optional[str] = None
|
||||
generation_mode: Optional[str] = None
|
||||
may_be_used_as_evidence: Optional[bool] = None
|
||||
reviewed_by: Optional[str] = None
|
||||
reviewed_at: Optional[datetime] = None
|
||||
# Anti-Fake-Evidence Phase 2: Four-Eyes
|
||||
approval_status: Optional[str] = None
|
||||
first_reviewer: Optional[str] = None
|
||||
first_reviewed_at: Optional[datetime] = None
|
||||
second_reviewer: Optional[str] = None
|
||||
second_reviewed_at: Optional[datetime] = None
|
||||
requires_four_eyes: Optional[bool] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
@@ -435,6 +453,25 @@ class AISystemListResponse(BaseModel):
|
||||
# Dashboard & Export Schemas
|
||||
# ============================================================================
|
||||
|
||||
class MultiDimensionalScore(BaseModel):
|
||||
"""Multi-dimensional compliance score (Anti-Fake-Evidence)."""
|
||||
requirement_coverage: float = 0.0 # % requirements with linked control
|
||||
evidence_strength: float = 0.0 # Weighted avg of evidence confidence (E0=0..E4=1)
|
||||
validation_quality: float = 0.0 # % evidence with truth_status >= validated_internal
|
||||
evidence_freshness: float = 0.0 # % evidence not expired + reviewed < 90 days
|
||||
control_effectiveness: float = 0.0 # Existing formula (pass + partial*0.5)
|
||||
overall_readiness: float = 0.0 # Weighted composite
|
||||
hard_blocks: List[str] = [] # Blocking issues preventing audit-readiness
|
||||
|
||||
|
||||
class StatusTransitionError(BaseModel):
|
||||
"""Error detail for forbidden control status transitions."""
|
||||
allowed: bool = False
|
||||
current_status: str
|
||||
requested_status: str
|
||||
violations: List[str] = []
|
||||
|
||||
|
||||
class DashboardResponse(BaseModel):
|
||||
compliance_score: float
|
||||
total_regulations: int
|
||||
@@ -447,6 +484,7 @@ class DashboardResponse(BaseModel):
|
||||
total_risks: int
|
||||
risks_by_level: Dict[str, int]
|
||||
recent_activity: List[Dict[str, Any]]
|
||||
multi_score: Optional[MultiDimensionalScore] = None
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
@@ -1939,3 +1977,111 @@ class TOMStatsResponse(BaseModel):
|
||||
implemented: int = 0
|
||||
partial: int = 0
|
||||
not_implemented: int = 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Assertion Schemas (Anti-Fake-Evidence Phase 2)
|
||||
# ============================================================================
|
||||
|
||||
class AssertionCreate(BaseModel):
|
||||
entity_type: str
|
||||
entity_id: str
|
||||
sentence_text: str
|
||||
assertion_type: Optional[str] = "assertion"
|
||||
evidence_ids: Optional[List[str]] = []
|
||||
normative_tier: Optional[str] = None
|
||||
|
||||
|
||||
class AssertionUpdate(BaseModel):
|
||||
sentence_text: Optional[str] = None
|
||||
assertion_type: Optional[str] = None
|
||||
evidence_ids: Optional[List[str]] = None
|
||||
normative_tier: Optional[str] = None
|
||||
confidence: Optional[float] = None
|
||||
|
||||
|
||||
class AssertionResponse(BaseModel):
|
||||
id: str
|
||||
tenant_id: Optional[str] = None
|
||||
entity_type: str
|
||||
entity_id: str
|
||||
sentence_text: str
|
||||
sentence_index: int = 0
|
||||
assertion_type: str = "assertion"
|
||||
evidence_ids: Optional[List[str]] = []
|
||||
confidence: float = 0.0
|
||||
normative_tier: Optional[str] = None
|
||||
verified_by: Optional[str] = None
|
||||
verified_at: Optional[datetime] = None
|
||||
created_at: Optional[datetime] = None
|
||||
updated_at: Optional[datetime] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class AssertionListResponse(BaseModel):
|
||||
assertions: List[AssertionResponse]
|
||||
total: int
|
||||
|
||||
|
||||
class AssertionSummaryResponse(BaseModel):
|
||||
total_assertions: int = 0
|
||||
total_facts: int = 0
|
||||
total_rationale: int = 0
|
||||
unverified_count: int = 0
|
||||
|
||||
|
||||
class AssertionExtractRequest(BaseModel):
|
||||
entity_type: str
|
||||
entity_id: str
|
||||
text: str
|
||||
|
||||
|
||||
class EvidenceRejectRequest(BaseModel):
|
||||
reviewed_by: str
|
||||
rejection_reason: Optional[str] = None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Traceability Matrix (Anti-Fake-Evidence Phase 4a)
|
||||
# ============================================================================
|
||||
|
||||
class TraceabilityAssertion(BaseModel):
|
||||
"""Single assertion linked to an evidence item."""
|
||||
id: str
|
||||
sentence_text: str
|
||||
assertion_type: str = "assertion"
|
||||
confidence: float = 0.0
|
||||
verified: bool = False
|
||||
|
||||
class TraceabilityEvidence(BaseModel):
|
||||
"""Evidence item with nested assertions."""
|
||||
id: str
|
||||
title: str
|
||||
evidence_type: str
|
||||
confidence_level: str = "E1"
|
||||
status: str = "valid"
|
||||
assertions: List[TraceabilityAssertion] = []
|
||||
|
||||
class TraceabilityCoverage(BaseModel):
|
||||
"""Coverage flags for a single control."""
|
||||
has_evidence: bool = False
|
||||
has_assertions: bool = False
|
||||
all_assertions_verified: bool = False
|
||||
min_confidence_level: Optional[str] = None
|
||||
|
||||
class TraceabilityControl(BaseModel):
|
||||
"""Control with nested evidence and coverage info."""
|
||||
id: str
|
||||
control_id: str
|
||||
title: str
|
||||
status: str = "planned"
|
||||
domain: str = "unknown"
|
||||
evidence: List[TraceabilityEvidence] = []
|
||||
coverage: TraceabilityCoverage = TraceabilityCoverage()
|
||||
|
||||
class TraceabilityMatrixResponse(BaseModel):
|
||||
"""Full traceability matrix: Controls → Evidence → Assertions."""
|
||||
controls: List[TraceabilityControl]
|
||||
summary: Dict[str, int]
|
||||
|
||||
@@ -8,12 +8,16 @@ from .models import (
|
||||
EvidenceDB,
|
||||
RiskDB,
|
||||
AuditExportDB,
|
||||
LLMGenerationAuditDB,
|
||||
AssertionDB,
|
||||
RegulationTypeEnum,
|
||||
ControlTypeEnum,
|
||||
ControlDomainEnum,
|
||||
RiskLevelEnum,
|
||||
EvidenceStatusEnum,
|
||||
ControlStatusEnum,
|
||||
EvidenceConfidenceEnum,
|
||||
EvidenceTruthStatusEnum,
|
||||
)
|
||||
from .repository import (
|
||||
RegulationRepository,
|
||||
@@ -33,6 +37,8 @@ __all__ = [
|
||||
"EvidenceDB",
|
||||
"RiskDB",
|
||||
"AuditExportDB",
|
||||
"LLMGenerationAuditDB",
|
||||
"AssertionDB",
|
||||
# Enums
|
||||
"RegulationTypeEnum",
|
||||
"ControlTypeEnum",
|
||||
@@ -40,6 +46,8 @@ __all__ = [
|
||||
"RiskLevelEnum",
|
||||
"EvidenceStatusEnum",
|
||||
"ControlStatusEnum",
|
||||
"EvidenceConfidenceEnum",
|
||||
"EvidenceTruthStatusEnum",
|
||||
# Repositories
|
||||
"RegulationRepository",
|
||||
"RequirementRepository",
|
||||
|
||||
@@ -65,6 +65,7 @@ class ControlStatusEnum(str, enum.Enum):
|
||||
FAIL = "fail" # Not passing
|
||||
NOT_APPLICABLE = "n/a" # Not applicable
|
||||
PLANNED = "planned" # Planned for implementation
|
||||
IN_PROGRESS = "in_progress" # Implementation in progress
|
||||
|
||||
|
||||
class RiskLevelEnum(str, enum.Enum):
|
||||
@@ -83,6 +84,26 @@ class EvidenceStatusEnum(str, enum.Enum):
|
||||
FAILED = "failed" # Failed validation
|
||||
|
||||
|
||||
class EvidenceConfidenceEnum(str, enum.Enum):
|
||||
"""Confidence level of evidence (Anti-Fake-Evidence)."""
|
||||
E0 = "E0" # Generated / no real evidence (LLM output, placeholder)
|
||||
E1 = "E1" # Uploaded but unreviewed (manual upload, no hash, no reviewer)
|
||||
E2 = "E2" # Reviewed internally (human reviewed, hash verified)
|
||||
E3 = "E3" # Observed by system (CI/CD pipeline, API with hash)
|
||||
E4 = "E4" # Validated by external auditor
|
||||
|
||||
|
||||
class EvidenceTruthStatusEnum(str, enum.Enum):
|
||||
"""Truth status lifecycle for evidence (Anti-Fake-Evidence)."""
|
||||
GENERATED = "generated"
|
||||
UPLOADED = "uploaded"
|
||||
OBSERVED = "observed"
|
||||
VALIDATED_INTERNAL = "validated_internal"
|
||||
REJECTED = "rejected"
|
||||
PROVIDED_TO_AUDITOR = "provided_to_auditor"
|
||||
ACCEPTED_BY_AUDITOR = "accepted_by_auditor"
|
||||
|
||||
|
||||
class ExportStatusEnum(str, enum.Enum):
|
||||
"""Status of audit export."""
|
||||
PENDING = "pending"
|
||||
@@ -239,6 +260,7 @@ class ControlDB(Base):
|
||||
# Status
|
||||
status = Column(Enum(ControlStatusEnum), default=ControlStatusEnum.PLANNED)
|
||||
status_notes = Column(Text)
|
||||
status_justification = Column(Text) # Required for n/a transitions
|
||||
|
||||
# Ownership & Review
|
||||
owner = Column(String(100)) # Responsible person/team
|
||||
@@ -321,6 +343,22 @@ class EvidenceDB(Base):
|
||||
ci_job_id = Column(String(100)) # CI/CD job reference
|
||||
uploaded_by = Column(String(100)) # User who uploaded
|
||||
|
||||
# Anti-Fake-Evidence: Confidence & Truth tracking
|
||||
confidence_level = Column(Enum(EvidenceConfidenceEnum), default=EvidenceConfidenceEnum.E1)
|
||||
truth_status = Column(Enum(EvidenceTruthStatusEnum), default=EvidenceTruthStatusEnum.UPLOADED)
|
||||
generation_mode = Column(String(100)) # e.g. "draft_assistance", "auto_generation"
|
||||
may_be_used_as_evidence = Column(Boolean, default=True)
|
||||
reviewed_by = Column(String(200))
|
||||
reviewed_at = Column(DateTime)
|
||||
|
||||
# Anti-Fake-Evidence Phase 2: Four-Eyes review
|
||||
approval_status = Column(String(30), default="none")
|
||||
first_reviewer = Column(String(200))
|
||||
first_reviewed_at = Column(DateTime)
|
||||
second_reviewer = Column(String(200))
|
||||
second_reviewed_at = Column(DateTime)
|
||||
requires_four_eyes = Column(Boolean, default=False)
|
||||
|
||||
# Timestamps
|
||||
collected_at = Column(DateTime, default=datetime.utcnow)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
@@ -332,6 +370,7 @@ class EvidenceDB(Base):
|
||||
__table_args__ = (
|
||||
Index('ix_evidence_control_type', 'control_id', 'evidence_type'),
|
||||
Index('ix_evidence_status', 'status'),
|
||||
Index('ix_evidence_approval_status', 'approval_status'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
@@ -1464,3 +1503,77 @@ class ISMSReadinessCheckDB(Base):
|
||||
|
||||
def __repr__(self):
|
||||
return f"<ISMSReadiness {self.check_date}: {self.overall_status}>"
|
||||
|
||||
|
||||
class LLMGenerationAuditDB(Base):
|
||||
"""
|
||||
Audit trail for LLM-generated content.
|
||||
|
||||
Every piece of content generated by an LLM is recorded here with its
|
||||
truth_status and may_be_used_as_evidence flag, ensuring transparency
|
||||
about what is real evidence vs. generated assistance.
|
||||
"""
|
||||
__tablename__ = 'compliance_llm_generation_audit'
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
tenant_id = Column(String(36), index=True)
|
||||
|
||||
entity_type = Column(String(50), nullable=False) # 'evidence', 'control', 'document'
|
||||
entity_id = Column(String(36)) # FK to generated entity
|
||||
generation_mode = Column(String(100), nullable=False) # 'draft_assistance', 'auto_generation'
|
||||
truth_status = Column(Enum(EvidenceTruthStatusEnum), nullable=False, default=EvidenceTruthStatusEnum.GENERATED)
|
||||
may_be_used_as_evidence = Column(Boolean, nullable=False, default=False)
|
||||
|
||||
llm_model = Column(String(100))
|
||||
llm_provider = Column(String(50)) # 'ollama', 'anthropic'
|
||||
prompt_hash = Column(String(64)) # SHA-256 of prompt
|
||||
input_summary = Column(Text)
|
||||
output_summary = Column(Text)
|
||||
extra_metadata = Column("metadata", JSON, default=dict)
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index('ix_llm_audit_entity', 'entity_type', 'entity_id'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<LLMGenerationAudit {self.entity_type}:{self.entity_id} mode={self.generation_mode}>"
|
||||
|
||||
|
||||
class AssertionDB(Base):
|
||||
"""
|
||||
Assertion tracking — separates claims from verified facts.
|
||||
|
||||
Each sentence from a control/evidence/document is stored here with its
|
||||
classification (assertion vs. fact vs. rationale) and optional evidence linkage.
|
||||
"""
|
||||
__tablename__ = 'compliance_assertions'
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
tenant_id = Column(String(36), index=True)
|
||||
|
||||
entity_type = Column(String(50), nullable=False) # 'control', 'evidence', 'document', 'obligation'
|
||||
entity_id = Column(String(36), nullable=False)
|
||||
sentence_text = Column(Text, nullable=False)
|
||||
sentence_index = Column(Integer, nullable=False, default=0)
|
||||
|
||||
assertion_type = Column(String(20), nullable=False, default='assertion') # 'assertion' | 'fact' | 'rationale'
|
||||
evidence_ids = Column(JSON, default=list)
|
||||
confidence = Column(Float, default=0.0)
|
||||
normative_tier = Column(String(20)) # 'pflicht' | 'empfehlung' | 'kann'
|
||||
|
||||
verified_by = Column(String(200))
|
||||
verified_at = Column(DateTime)
|
||||
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
Index('ix_assertion_entity', 'entity_type', 'entity_id'),
|
||||
Index('ix_assertion_type', 'assertion_type'),
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Assertion {self.assertion_type}: {self.sentence_text[:50]}>"
|
||||
|
||||
@@ -487,6 +487,137 @@ class ControlRepository:
|
||||
"compliance_score": round(score, 1),
|
||||
}
|
||||
|
||||
def get_multi_dimensional_score(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Calculate multi-dimensional compliance score (Anti-Fake-Evidence).
|
||||
|
||||
Returns 6 dimensions + hard_blocks + overall_readiness.
|
||||
"""
|
||||
from .models import (
|
||||
EvidenceDB, RequirementDB, ControlMappingDB,
|
||||
EvidenceConfidenceEnum, EvidenceTruthStatusEnum,
|
||||
)
|
||||
|
||||
# Weight map for confidence levels
|
||||
conf_weights = {"E0": 0.0, "E1": 0.25, "E2": 0.5, "E3": 0.75, "E4": 1.0}
|
||||
validated_statuses = {"validated_internal", "accepted_by_auditor", "provided_to_auditor"}
|
||||
|
||||
controls = self.get_all()
|
||||
total_controls = len(controls)
|
||||
|
||||
if total_controls == 0:
|
||||
return {
|
||||
"requirement_coverage": 0.0,
|
||||
"evidence_strength": 0.0,
|
||||
"validation_quality": 0.0,
|
||||
"evidence_freshness": 0.0,
|
||||
"control_effectiveness": 0.0,
|
||||
"overall_readiness": 0.0,
|
||||
"hard_blocks": ["Keine Controls vorhanden"],
|
||||
}
|
||||
|
||||
# 1. requirement_coverage: % requirements linked to at least one control
|
||||
total_reqs = self.db.query(func.count(RequirementDB.id)).scalar() or 0
|
||||
linked_reqs = (
|
||||
self.db.query(func.count(func.distinct(ControlMappingDB.requirement_id)))
|
||||
.scalar() or 0
|
||||
)
|
||||
requirement_coverage = (linked_reqs / total_reqs * 100) if total_reqs > 0 else 0.0
|
||||
|
||||
# 2. evidence_strength: weighted average of evidence confidence
|
||||
all_evidence = self.db.query(EvidenceDB).all()
|
||||
if all_evidence:
|
||||
total_weight = 0.0
|
||||
for e in all_evidence:
|
||||
conf_val = e.confidence_level.value if e.confidence_level else "E1"
|
||||
total_weight += conf_weights.get(conf_val, 0.25)
|
||||
evidence_strength = (total_weight / len(all_evidence)) * 100
|
||||
else:
|
||||
evidence_strength = 0.0
|
||||
|
||||
# 3. validation_quality: % evidence with truth_status >= validated_internal
|
||||
if all_evidence:
|
||||
validated_count = sum(
|
||||
1 for e in all_evidence
|
||||
if (e.truth_status.value if e.truth_status else "uploaded") in validated_statuses
|
||||
)
|
||||
validation_quality = (validated_count / len(all_evidence)) * 100
|
||||
else:
|
||||
validation_quality = 0.0
|
||||
|
||||
# 4. evidence_freshness: % evidence not expired and reviewed < 90 days
|
||||
now = datetime.now()
|
||||
if all_evidence:
|
||||
fresh_count = 0
|
||||
for e in all_evidence:
|
||||
is_expired = e.valid_until and e.valid_until < now
|
||||
is_stale = e.reviewed_at and (now - e.reviewed_at).days > 90 if hasattr(e, 'reviewed_at') and e.reviewed_at else False
|
||||
if not is_expired and not is_stale:
|
||||
fresh_count += 1
|
||||
evidence_freshness = (fresh_count / len(all_evidence)) * 100
|
||||
else:
|
||||
evidence_freshness = 0.0
|
||||
|
||||
# 5. control_effectiveness: existing formula
|
||||
passed = sum(1 for c in controls if c.status == ControlStatusEnum.PASS)
|
||||
partial = sum(1 for c in controls if c.status == ControlStatusEnum.PARTIAL)
|
||||
control_effectiveness = ((passed + partial * 0.5) / total_controls) * 100
|
||||
|
||||
# 6. overall_readiness: weighted composite
|
||||
overall_readiness = (
|
||||
0.20 * requirement_coverage +
|
||||
0.25 * evidence_strength +
|
||||
0.20 * validation_quality +
|
||||
0.10 * evidence_freshness +
|
||||
0.25 * control_effectiveness
|
||||
)
|
||||
|
||||
# Hard blocks
|
||||
hard_blocks = []
|
||||
|
||||
# Critical controls without any evidence
|
||||
critical_no_evidence = []
|
||||
for c in controls:
|
||||
if c.status in (ControlStatusEnum.PASS, ControlStatusEnum.PARTIAL):
|
||||
evidence_for_ctrl = [e for e in all_evidence if e.control_id == c.id]
|
||||
if not evidence_for_ctrl:
|
||||
critical_no_evidence.append(c.control_id)
|
||||
if critical_no_evidence:
|
||||
hard_blocks.append(
|
||||
f"{len(critical_no_evidence)} Controls mit Status pass/partial haben keine Evidence: "
|
||||
f"{', '.join(critical_no_evidence[:5])}"
|
||||
)
|
||||
|
||||
# Controls with only E0/E1 evidence claiming pass
|
||||
weak_evidence_pass = []
|
||||
for c in controls:
|
||||
if c.status == ControlStatusEnum.PASS:
|
||||
evidence_for_ctrl = [e for e in all_evidence if e.control_id == c.id]
|
||||
if evidence_for_ctrl:
|
||||
max_conf = max(
|
||||
conf_weights.get(
|
||||
e.confidence_level.value if e.confidence_level else "E1", 0.25
|
||||
)
|
||||
for e in evidence_for_ctrl
|
||||
)
|
||||
if max_conf < 0.5: # Only E0 or E1
|
||||
weak_evidence_pass.append(c.control_id)
|
||||
if weak_evidence_pass:
|
||||
hard_blocks.append(
|
||||
f"{len(weak_evidence_pass)} Controls auf 'pass' haben nur E0/E1-Evidence: "
|
||||
f"{', '.join(weak_evidence_pass[:5])}"
|
||||
)
|
||||
|
||||
return {
|
||||
"requirement_coverage": round(requirement_coverage, 1),
|
||||
"evidence_strength": round(evidence_strength, 1),
|
||||
"validation_quality": round(validation_quality, 1),
|
||||
"evidence_freshness": round(evidence_freshness, 1),
|
||||
"control_effectiveness": round(control_effectiveness, 1),
|
||||
"overall_readiness": round(overall_readiness, 1),
|
||||
"hard_blocks": hard_blocks,
|
||||
}
|
||||
|
||||
|
||||
class ControlMappingRepository:
|
||||
"""Repository for requirement-control mappings."""
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Assertion Engine — splits text into sentences and classifies each.
|
||||
|
||||
Each sentence is tagged as:
|
||||
- assertion: normative statement (pflicht / empfehlung / kann)
|
||||
- fact: references concrete evidence artifacts
|
||||
- rationale: explains why something is required
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from .normative_patterns import (
|
||||
PFLICHT_RE, EMPFEHLUNG_RE, KANN_RE, RATIONALE_RE, EVIDENCE_RE,
|
||||
)
|
||||
|
||||
# Sentence splitter: period/excl/question followed by space+uppercase, or newlines
|
||||
_SENTENCE_SPLIT = re.compile(r'(?<=[.!?])\s+(?=[A-ZÄÖÜ])|(?:\n\s*\n)')
|
||||
|
||||
|
||||
def extract_assertions(
|
||||
text: str,
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
tenant_id: Optional[str] = None,
|
||||
) -> list[dict]:
|
||||
"""Split *text* into sentences and classify each one.
|
||||
|
||||
Returns a list of dicts ready for AssertionDB creation.
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return []
|
||||
|
||||
sentences = _SENTENCE_SPLIT.split(text.strip())
|
||||
results: list[dict] = []
|
||||
|
||||
for idx, raw in enumerate(sentences):
|
||||
sentence = raw.strip()
|
||||
if not sentence or len(sentence) < 5:
|
||||
continue
|
||||
|
||||
assertion_type, normative_tier = _classify_sentence(sentence)
|
||||
|
||||
results.append({
|
||||
"tenant_id": tenant_id,
|
||||
"entity_type": entity_type,
|
||||
"entity_id": entity_id,
|
||||
"sentence_text": sentence,
|
||||
"sentence_index": idx,
|
||||
"assertion_type": assertion_type,
|
||||
"normative_tier": normative_tier,
|
||||
"evidence_ids": [],
|
||||
"confidence": 0.0,
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _classify_sentence(sentence: str) -> tuple[str, Optional[str]]:
|
||||
"""Return (assertion_type, normative_tier) for a single sentence."""
|
||||
|
||||
# 1. Check for evidence/fact keywords first
|
||||
if EVIDENCE_RE.search(sentence):
|
||||
return ("fact", None)
|
||||
|
||||
# 2. Check for rationale
|
||||
normative_count = len(PFLICHT_RE.findall(sentence)) + len(EMPFEHLUNG_RE.findall(sentence)) + len(KANN_RE.findall(sentence))
|
||||
rationale_count = len(RATIONALE_RE.findall(sentence))
|
||||
if rationale_count > 0 and rationale_count >= normative_count:
|
||||
return ("rationale", None)
|
||||
|
||||
# 3. Normative classification
|
||||
if PFLICHT_RE.search(sentence):
|
||||
return ("assertion", "pflicht")
|
||||
if EMPFEHLUNG_RE.search(sentence):
|
||||
return ("assertion", "empfehlung")
|
||||
if KANN_RE.search(sentence):
|
||||
return ("assertion", "kann")
|
||||
|
||||
# 4. Default: unclassified assertion
|
||||
return ("assertion", None)
|
||||
@@ -493,6 +493,9 @@ class GeneratedControl:
|
||||
applicable_industries: Optional[list] = None # e.g. ["all"] or ["Telekommunikation", "Energie"]
|
||||
applicable_company_size: Optional[list] = None # e.g. ["all"] or ["medium", "large", "enterprise"]
|
||||
scope_conditions: Optional[dict] = None # e.g. {"requires_any": ["uses_ai"], "description": "..."}
|
||||
# Anti-Fake-Evidence: truth tracking for generated controls
|
||||
truth_status: str = "generated"
|
||||
may_be_used_as_evidence: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -781,10 +784,23 @@ REFORM_SYSTEM_PROMPT = """Du bist ein Security-Compliance-Experte. Deine Aufgabe
|
||||
Security Controls zu formulieren. Du formulierst IMMER in eigenen Worten.
|
||||
KOPIERE KEINE Sätze aus dem Quelltext. Verwende eigene Begriffe und Struktur.
|
||||
NENNE NICHT die Quelle. Keine proprietären Bezeichner.
|
||||
|
||||
WICHTIG — Truthfulness-Guardrail:
|
||||
Deine Ausgabe ist ein ENTWURF. Formuliere NIEMALS Behauptungen über bereits erfolgte Umsetzung.
|
||||
Verwende NICHT: "ist compliant", "erfüllt vollständig", "wurde geprüft", "wurde umgesetzt",
|
||||
"ist auditiert", "vollständig implementiert", "nachweislich konform".
|
||||
Verwende stattdessen: "soll umsetzen", "ist vorgesehen", "muss implementiert werden".
|
||||
|
||||
Antworte NUR mit validem JSON. Bei mehreren Controls antworte mit einem JSON-Array."""
|
||||
|
||||
STRUCTURE_SYSTEM_PROMPT = """Du bist ein Security-Compliance-Experte. Strukturiere den gegebenen Text
|
||||
als praxisorientiertes Security Control. Erstelle eine verständliche, umsetzbare Formulierung.
|
||||
|
||||
WICHTIG — Truthfulness-Guardrail:
|
||||
Deine Ausgabe ist ein ENTWURF. Formuliere NIEMALS Behauptungen über bereits erfolgte Umsetzung.
|
||||
Verwende NICHT: "ist compliant", "erfüllt vollständig", "wurde geprüft", "wurde umgesetzt".
|
||||
Verwende stattdessen: "soll umsetzen", "ist vorgesehen", "muss implementiert werden".
|
||||
|
||||
Antworte NUR mit validem JSON. Bei mehreren Controls antworte mit einem JSON-Array."""
|
||||
|
||||
# Shared applicability prompt block — appended to all generation prompts (v3)
|
||||
@@ -1877,7 +1893,38 @@ Kategorien: {CATEGORY_LIST_STR}"""
|
||||
)
|
||||
self.db.commit()
|
||||
row = result.fetchone()
|
||||
return str(row[0]) if row else None
|
||||
control_uuid = str(row[0]) if row else None
|
||||
|
||||
# Anti-Fake-Evidence: Record LLM audit trail for generated control
|
||||
if control_uuid:
|
||||
try:
|
||||
self.db.execute(
|
||||
text("""
|
||||
INSERT INTO compliance_llm_generation_audit (
|
||||
entity_type, entity_id, generation_mode,
|
||||
truth_status, may_be_used_as_evidence,
|
||||
llm_model, llm_provider,
|
||||
input_summary, output_summary
|
||||
) VALUES (
|
||||
'control', :entity_id, 'auto_generation',
|
||||
'generated', FALSE,
|
||||
:llm_model, :llm_provider,
|
||||
:input_summary, :output_summary
|
||||
)
|
||||
"""),
|
||||
{
|
||||
"entity_id": control_uuid,
|
||||
"llm_model": ANTHROPIC_MODEL if ANTHROPIC_API_KEY else OLLAMA_MODEL,
|
||||
"llm_provider": "anthropic" if ANTHROPIC_API_KEY else "ollama",
|
||||
"input_summary": f"Control generation for {control.control_id}",
|
||||
"output_summary": control.title[:500] if control.title else None,
|
||||
},
|
||||
)
|
||||
self.db.commit()
|
||||
except Exception as audit_err:
|
||||
logger.warning("Failed to create LLM audit record: %s", audit_err)
|
||||
|
||||
return control_uuid
|
||||
except Exception as e:
|
||||
logger.error("Failed to store control %s: %s", control.control_id, e)
|
||||
self.db.rollback()
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
Control Status Transition State Machine.
|
||||
|
||||
Enforces that controls cannot be set to "pass" without sufficient evidence.
|
||||
Prevents Compliance-Theater where controls claim compliance without real proof.
|
||||
|
||||
Transition rules:
|
||||
planned → in_progress : always allowed
|
||||
in_progress → pass : requires ≥1 evidence with confidence ≥ E2 and
|
||||
truth_status in (uploaded, observed, validated_internal)
|
||||
in_progress → partial : requires ≥1 evidence (any level)
|
||||
pass → fail : always allowed (degradation)
|
||||
any → n/a : requires status_justification
|
||||
any → planned : always allowed (reset)
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from ..db.models import EvidenceDB
|
||||
|
||||
|
||||
# Confidence level ordering for comparisons
|
||||
CONFIDENCE_ORDER = {"E0": 0, "E1": 1, "E2": 2, "E3": 3, "E4": 4}
|
||||
|
||||
# Truth statuses that qualify as "real" evidence for pass transitions
|
||||
VALID_TRUTH_STATUSES = {"uploaded", "observed", "validated_internal", "accepted_by_auditor", "provided_to_auditor"}
|
||||
|
||||
|
||||
def validate_transition(
|
||||
current_status: str,
|
||||
new_status: str,
|
||||
evidence_list: Optional[List[EvidenceDB]] = None,
|
||||
status_justification: Optional[str] = None,
|
||||
bypass_for_auto_updater: bool = False,
|
||||
) -> Tuple[bool, List[str]]:
|
||||
"""
|
||||
Validate whether a control status transition is allowed.
|
||||
|
||||
Args:
|
||||
current_status: Current control status value (e.g. "planned", "pass")
|
||||
new_status: Requested new status
|
||||
evidence_list: List of EvidenceDB objects linked to this control
|
||||
status_justification: Text justification (required for n/a transitions)
|
||||
bypass_for_auto_updater: If True, skip evidence checks (used by CI/CD auto-updater
|
||||
which creates evidence atomically with status change)
|
||||
|
||||
Returns:
|
||||
Tuple of (allowed: bool, violations: list[str])
|
||||
"""
|
||||
violations: List[str] = []
|
||||
evidence_list = evidence_list or []
|
||||
|
||||
# Same status → no-op, always allowed
|
||||
if current_status == new_status:
|
||||
return True, []
|
||||
|
||||
# Reset to planned is always allowed
|
||||
if new_status == "planned":
|
||||
return True, []
|
||||
|
||||
# n/a requires justification
|
||||
if new_status == "n/a":
|
||||
if not status_justification or not status_justification.strip():
|
||||
violations.append("Transition to 'n/a' requires a status_justification explaining why this control is not applicable.")
|
||||
return len(violations) == 0, violations
|
||||
|
||||
# Degradation: pass → fail is always allowed
|
||||
if current_status == "pass" and new_status == "fail":
|
||||
return True, []
|
||||
|
||||
# planned → in_progress: always allowed
|
||||
if current_status == "planned" and new_status == "in_progress":
|
||||
return True, []
|
||||
|
||||
# in_progress → partial: needs at least 1 evidence
|
||||
if new_status == "partial":
|
||||
if not bypass_for_auto_updater and len(evidence_list) == 0:
|
||||
violations.append("Transition to 'partial' requires at least 1 evidence record.")
|
||||
return len(violations) == 0, violations
|
||||
|
||||
# in_progress → pass: strict requirements
|
||||
if new_status == "pass":
|
||||
if bypass_for_auto_updater:
|
||||
return True, []
|
||||
|
||||
if len(evidence_list) == 0:
|
||||
violations.append("Transition to 'pass' requires at least 1 evidence record.")
|
||||
return False, violations
|
||||
|
||||
# Check for at least one qualifying evidence
|
||||
has_qualifying = False
|
||||
for e in evidence_list:
|
||||
conf = getattr(e, "confidence_level", None)
|
||||
truth = getattr(e, "truth_status", None)
|
||||
|
||||
# Get string values from enum or string
|
||||
conf_val = conf.value if hasattr(conf, "value") else str(conf) if conf else "E1"
|
||||
truth_val = truth.value if hasattr(truth, "value") else str(truth) if truth else "uploaded"
|
||||
|
||||
if CONFIDENCE_ORDER.get(conf_val, 1) >= CONFIDENCE_ORDER["E2"] and truth_val in VALID_TRUTH_STATUSES:
|
||||
has_qualifying = True
|
||||
break
|
||||
|
||||
if not has_qualifying:
|
||||
violations.append(
|
||||
"Transition to 'pass' requires at least 1 evidence with confidence >= E2 "
|
||||
"and truth_status in (uploaded, observed, validated_internal, accepted_by_auditor). "
|
||||
"Current evidence does not meet this threshold."
|
||||
)
|
||||
|
||||
return len(violations) == 0, violations
|
||||
|
||||
# in_progress → fail: always allowed
|
||||
if new_status == "fail":
|
||||
return True, []
|
||||
|
||||
# Any other transition from planned/fail to pass requires going through in_progress
|
||||
if current_status in ("planned", "fail") and new_status == "pass":
|
||||
if bypass_for_auto_updater:
|
||||
return True, []
|
||||
violations.append(
|
||||
f"Direct transition from '{current_status}' to 'pass' is not allowed. "
|
||||
f"Move to 'in_progress' first, then to 'pass' with qualifying evidence."
|
||||
)
|
||||
return False, violations
|
||||
|
||||
# Default: allow other transitions (e.g. fail → partial, partial → pass)
|
||||
# For partial → pass, apply the same evidence checks
|
||||
if current_status == "partial" and new_status == "pass":
|
||||
if bypass_for_auto_updater:
|
||||
return True, []
|
||||
|
||||
has_qualifying = False
|
||||
for e in evidence_list:
|
||||
conf = getattr(e, "confidence_level", None)
|
||||
truth = getattr(e, "truth_status", None)
|
||||
conf_val = conf.value if hasattr(conf, "value") else str(conf) if conf else "E1"
|
||||
truth_val = truth.value if hasattr(truth, "value") else str(truth) if truth else "uploaded"
|
||||
|
||||
if CONFIDENCE_ORDER.get(conf_val, 1) >= CONFIDENCE_ORDER["E2"] and truth_val in VALID_TRUTH_STATUSES:
|
||||
has_qualifying = True
|
||||
break
|
||||
|
||||
if not has_qualifying:
|
||||
violations.append(
|
||||
"Transition from 'partial' to 'pass' requires at least 1 evidence with confidence >= E2 "
|
||||
"and truth_status in (uploaded, observed, validated_internal, accepted_by_auditor)."
|
||||
)
|
||||
return len(violations) == 0, violations
|
||||
|
||||
# All other transitions allowed
|
||||
return True, []
|
||||
@@ -52,64 +52,18 @@ ANTHROPIC_API_URL = "https://api.anthropic.com/v1"
|
||||
# Tier 2: Empfehlung (recommendation) — weaker normative signals
|
||||
# Tier 3: Kann (optional/permissive) — permissive signals
|
||||
# Nothing is rejected — everything is classified.
|
||||
#
|
||||
# Patterns are defined in normative_patterns.py and imported here
|
||||
# with local aliases for backward compatibility.
|
||||
|
||||
_PFLICHT_SIGNALS = [
|
||||
# Deutsche modale Pflichtformulierungen
|
||||
r"\bmüssen\b", r"\bmuss\b", r"\bhat\s+sicherzustellen\b",
|
||||
r"\bhaben\s+sicherzustellen\b", r"\bsind\s+verpflichtet\b",
|
||||
r"\bist\s+verpflichtet\b",
|
||||
# "ist zu prüfen", "sind zu dokumentieren" (direkt)
|
||||
r"\bist\s+zu\s+\w+en\b", r"\bsind\s+zu\s+\w+en\b",
|
||||
r"\bhat\s+zu\s+\w+en\b", r"\bhaben\s+zu\s+\w+en\b",
|
||||
# "ist festzustellen", "sind vorzunehmen" (Compound-Verben, eingebettetes zu)
|
||||
r"\bist\s+\w+zu\w+en\b", r"\bsind\s+\w+zu\w+en\b",
|
||||
# "ist zusätzlich zu prüfen", "sind regelmäßig zu überwachen" (Adverb dazwischen)
|
||||
r"\bist\s+\w+\s+zu\s+\w+en\b", r"\bsind\s+\w+\s+zu\s+\w+en\b",
|
||||
r"\bhat\s+\w+\s+zu\s+\w+en\b", r"\bhaben\s+\w+\s+zu\s+\w+en\b",
|
||||
# Englische Pflicht-Signale
|
||||
r"\bshall\b", r"\bmust\b", r"\brequired\b",
|
||||
# Compound-Infinitive (Gerundivum): mitzuteilen, anzuwenden, bereitzustellen
|
||||
r"\b\w+zuteilen\b", r"\b\w+zuwenden\b", r"\b\w+zustellen\b", r"\b\w+zulegen\b",
|
||||
r"\b\w+zunehmen\b", r"\b\w+zuführen\b", r"\b\w+zuhalten\b", r"\b\w+zusetzen\b",
|
||||
r"\b\w+zuweisen\b", r"\b\w+zuordnen\b", r"\b\w+zufügen\b", r"\b\w+zugeben\b",
|
||||
# Breites Pattern: "ist ... [bis 80 Zeichen] ... zu + Infinitiv"
|
||||
r"\bist\b.{1,80}\bzu\s+\w+en\b", r"\bsind\b.{1,80}\bzu\s+\w+en\b",
|
||||
]
|
||||
_PFLICHT_RE = re.compile("|".join(_PFLICHT_SIGNALS), re.IGNORECASE)
|
||||
|
||||
_EMPFEHLUNG_SIGNALS = [
|
||||
# Modale Verben (schwaecher als "muss")
|
||||
r"\bsoll\b", r"\bsollen\b", r"\bsollte\b", r"\bsollten\b",
|
||||
r"\bgewährleisten\b", r"\bsicherstellen\b",
|
||||
# Englische Empfehlungs-Signale
|
||||
r"\bshould\b", r"\bensure\b", r"\brecommend\w*\b",
|
||||
# Haeufige normative Infinitive (ohne Hilfsverb, als Empfehlung)
|
||||
r"\bnachweisen\b", r"\beinhalten\b", r"\bunterlassen\b", r"\bwahren\b",
|
||||
r"\bdokumentieren\b", r"\bimplementieren\b", r"\büberprüfen\b", r"\büberwachen\b",
|
||||
# Pruefanweisungen als normative Aussage
|
||||
r"\bprüfen,\s+ob\b", r"\bkontrollieren,\s+ob\b",
|
||||
]
|
||||
_EMPFEHLUNG_RE = re.compile("|".join(_EMPFEHLUNG_SIGNALS), re.IGNORECASE)
|
||||
|
||||
_KANN_SIGNALS = [
|
||||
r"\bkann\b", r"\bkönnen\b", r"\bdarf\b", r"\bdürfen\b",
|
||||
r"\bmay\b", r"\boptional\b",
|
||||
]
|
||||
_KANN_RE = re.compile("|".join(_KANN_SIGNALS), re.IGNORECASE)
|
||||
|
||||
# Union of all normative signals (for backward-compatible has_normative_signal flag)
|
||||
_NORMATIVE_RE = re.compile(
|
||||
"|".join(_PFLICHT_SIGNALS + _EMPFEHLUNG_SIGNALS + _KANN_SIGNALS),
|
||||
re.IGNORECASE,
|
||||
from .normative_patterns import (
|
||||
PFLICHT_RE as _PFLICHT_RE,
|
||||
EMPFEHLUNG_RE as _EMPFEHLUNG_RE,
|
||||
KANN_RE as _KANN_RE,
|
||||
NORMATIVE_RE as _NORMATIVE_RE,
|
||||
RATIONALE_RE as _RATIONALE_RE,
|
||||
)
|
||||
|
||||
_RATIONALE_SIGNALS = [
|
||||
r"\bda\s+", r"\bweil\b", r"\bgrund\b", r"\berwägung",
|
||||
r"\bbecause\b", r"\breason\b", r"\brationale\b",
|
||||
r"\bkönnen\s+.*\s+verursachen\b", r"\bführt\s+zu\b",
|
||||
]
|
||||
_RATIONALE_RE = re.compile("|".join(_RATIONALE_SIGNALS), re.IGNORECASE)
|
||||
|
||||
_TEST_SIGNALS = [
|
||||
r"\btesten\b", r"\btest\b", r"\bprüfung\b", r"\bprüfen\b",
|
||||
r"\bgetestet\b", r"\bwirksamkeit\b", r"\baudit\b",
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Shared normative language patterns for assertion classification.
|
||||
|
||||
Extracted from decomposition_pass.py for reuse in the assertion engine.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
_PFLICHT_SIGNALS = [
|
||||
r"\bmüssen\b", r"\bmuss\b", r"\bhat\s+sicherzustellen\b",
|
||||
r"\bhaben\s+sicherzustellen\b", r"\bsind\s+verpflichtet\b",
|
||||
r"\bist\s+verpflichtet\b",
|
||||
r"\bist\s+zu\s+\w+en\b", r"\bsind\s+zu\s+\w+en\b",
|
||||
r"\bhat\s+zu\s+\w+en\b", r"\bhaben\s+zu\s+\w+en\b",
|
||||
r"\bist\s+\w+zu\w+en\b", r"\bsind\s+\w+zu\w+en\b",
|
||||
r"\bist\s+\w+\s+zu\s+\w+en\b", r"\bsind\s+\w+\s+zu\s+\w+en\b",
|
||||
r"\bhat\s+\w+\s+zu\s+\w+en\b", r"\bhaben\s+\w+\s+zu\s+\w+en\b",
|
||||
r"\bshall\b", r"\bmust\b", r"\brequired\b",
|
||||
r"\b\w+zuteilen\b", r"\b\w+zuwenden\b", r"\b\w+zustellen\b", r"\b\w+zulegen\b",
|
||||
r"\b\w+zunehmen\b", r"\b\w+zuführen\b", r"\b\w+zuhalten\b", r"\b\w+zusetzen\b",
|
||||
r"\b\w+zuweisen\b", r"\b\w+zuordnen\b", r"\b\w+zufügen\b", r"\b\w+zugeben\b",
|
||||
r"\bist\b.{1,80}\bzu\s+\w+en\b", r"\bsind\b.{1,80}\bzu\s+\w+en\b",
|
||||
]
|
||||
PFLICHT_RE = re.compile("|".join(_PFLICHT_SIGNALS), re.IGNORECASE)
|
||||
|
||||
_EMPFEHLUNG_SIGNALS = [
|
||||
r"\bsoll\b", r"\bsollen\b", r"\bsollte\b", r"\bsollten\b",
|
||||
r"\bgewährleisten\b", r"\bsicherstellen\b",
|
||||
r"\bshould\b", r"\bensure\b", r"\brecommend\w*\b",
|
||||
r"\bnachweisen\b", r"\beinhalten\b", r"\bunterlassen\b", r"\bwahren\b",
|
||||
r"\bdokumentieren\b", r"\bimplementieren\b", r"\büberprüfen\b", r"\büberwachen\b",
|
||||
r"\bprüfen,\s+ob\b", r"\bkontrollieren,\s+ob\b",
|
||||
]
|
||||
EMPFEHLUNG_RE = re.compile("|".join(_EMPFEHLUNG_SIGNALS), re.IGNORECASE)
|
||||
|
||||
_KANN_SIGNALS = [
|
||||
r"\bkann\b", r"\bkönnen\b", r"\bdarf\b", r"\bdürfen\b",
|
||||
r"\bmay\b", r"\boptional\b",
|
||||
]
|
||||
KANN_RE = re.compile("|".join(_KANN_SIGNALS), re.IGNORECASE)
|
||||
|
||||
NORMATIVE_RE = re.compile(
|
||||
"|".join(_PFLICHT_SIGNALS + _EMPFEHLUNG_SIGNALS + _KANN_SIGNALS),
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_RATIONALE_SIGNALS = [
|
||||
r"\bda\s+", r"\bweil\b", r"\bgrund\b", r"\berwägung",
|
||||
r"\bbecause\b", r"\breason\b", r"\brationale\b",
|
||||
r"\bkönnen\s+.*\s+verursachen\b", r"\bführt\s+zu\b",
|
||||
]
|
||||
RATIONALE_RE = re.compile("|".join(_RATIONALE_SIGNALS), re.IGNORECASE)
|
||||
|
||||
# Evidence-related keywords (for fact detection)
|
||||
_EVIDENCE_KEYWORDS = [
|
||||
r"\bnachweis\b", r"\bzertifikat\b", r"\baudit.report\b",
|
||||
r"\bprotokoll\b", r"\bdokumentation\b", r"\bbericht\b",
|
||||
r"\bcertificate\b", r"\bevidence\b", r"\bproof\b",
|
||||
]
|
||||
EVIDENCE_RE = re.compile("|".join(_EVIDENCE_KEYWORDS), re.IGNORECASE)
|
||||
@@ -0,0 +1,125 @@
|
||||
-- Migration 076: Anti-Fake-Evidence Guardrails (Phase 1)
|
||||
--
|
||||
-- Prevents "Compliance-Theater": generated content passed off as real evidence,
|
||||
-- controls without evidence marked as "pass", unvalidated 100% compliance claims.
|
||||
--
|
||||
-- Changes:
|
||||
-- 1. New ENUM types for evidence confidence + truth status
|
||||
-- 2. New columns on compliance_evidence (confidence, truth, review tracking)
|
||||
-- 3. New value 'in_progress' for controlstatusenum
|
||||
-- 4. status_justification column on compliance_controls
|
||||
-- 5. New table compliance_llm_generation_audit
|
||||
-- 6. Backfill existing evidence based on source
|
||||
-- 7. Indexes on new columns
|
||||
|
||||
-- ============================================================================
|
||||
-- 1. New ENUM types
|
||||
-- ============================================================================
|
||||
|
||||
-- NOTE: CREATE TYPE cannot run inside a transaction block when combined with
|
||||
-- ALTER TYPE ... ADD VALUE. Each statement here is auto-committed separately
|
||||
-- when executed outside a transaction (which is the default for psql scripts).
|
||||
|
||||
CREATE TYPE evidence_confidence_level AS ENUM (
|
||||
'E0', -- Generated / no real evidence (LLM output, placeholder)
|
||||
'E1', -- Uploaded but unreviewed (manual upload, no hash, no reviewer)
|
||||
'E2', -- Reviewed internally (human reviewed, hash verified)
|
||||
'E3', -- Observed by system (CI/CD pipeline, API with hash)
|
||||
'E4' -- Validated by external auditor
|
||||
);
|
||||
|
||||
CREATE TYPE evidence_truth_status AS ENUM (
|
||||
'generated', -- Created by LLM / system generation
|
||||
'uploaded', -- Manually uploaded by user
|
||||
'observed', -- Automatically observed (CI/CD, monitoring)
|
||||
'validated_internal', -- Reviewed + approved by internal reviewer
|
||||
'rejected', -- Reviewed and rejected
|
||||
'provided_to_auditor', -- Shared with external auditor
|
||||
'accepted_by_auditor' -- Accepted by external auditor
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 2. Add 'in_progress' to controlstatusenum
|
||||
-- ============================================================================
|
||||
-- ALTER TYPE ... ADD VALUE cannot run inside a transaction.
|
||||
|
||||
ALTER TYPE controlstatusenum ADD VALUE IF NOT EXISTS 'in_progress';
|
||||
|
||||
-- ============================================================================
|
||||
-- 3. New columns on compliance_evidence
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE compliance_evidence
|
||||
ADD COLUMN IF NOT EXISTS confidence_level evidence_confidence_level DEFAULT 'E1',
|
||||
ADD COLUMN IF NOT EXISTS truth_status evidence_truth_status DEFAULT 'uploaded',
|
||||
ADD COLUMN IF NOT EXISTS generation_mode VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS may_be_used_as_evidence BOOLEAN DEFAULT TRUE,
|
||||
ADD COLUMN IF NOT EXISTS reviewed_by VARCHAR(200),
|
||||
ADD COLUMN IF NOT EXISTS reviewed_at TIMESTAMPTZ;
|
||||
|
||||
-- ============================================================================
|
||||
-- 4. status_justification on compliance_controls
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE compliance_controls
|
||||
ADD COLUMN IF NOT EXISTS status_justification TEXT;
|
||||
|
||||
-- ============================================================================
|
||||
-- 5. LLM Generation Audit table
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compliance_llm_generation_audit (
|
||||
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
||||
tenant_id VARCHAR(36),
|
||||
entity_type VARCHAR(50) NOT NULL, -- 'evidence', 'control', 'document', ...
|
||||
entity_id VARCHAR(36), -- FK to the generated entity
|
||||
generation_mode VARCHAR(100) NOT NULL, -- 'draft_assistance', 'auto_generation', ...
|
||||
truth_status evidence_truth_status NOT NULL DEFAULT 'generated',
|
||||
may_be_used_as_evidence BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
llm_model VARCHAR(100),
|
||||
llm_provider VARCHAR(50), -- 'ollama', 'anthropic', ...
|
||||
prompt_hash VARCHAR(64), -- SHA-256 of the prompt
|
||||
input_summary TEXT, -- Truncated input for auditability
|
||||
output_summary TEXT, -- Truncated output for auditability
|
||||
metadata JSONB DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 6. Backfill existing evidence based on source
|
||||
-- ============================================================================
|
||||
|
||||
-- CI pipeline evidence → E3 + observed
|
||||
UPDATE compliance_evidence
|
||||
SET confidence_level = 'E3',
|
||||
truth_status = 'observed'
|
||||
WHERE source = 'ci_pipeline'
|
||||
AND confidence_level = 'E1';
|
||||
|
||||
-- API evidence → E3 + observed
|
||||
UPDATE compliance_evidence
|
||||
SET confidence_level = 'E3',
|
||||
truth_status = 'observed'
|
||||
WHERE source = 'api'
|
||||
AND confidence_level = 'E1';
|
||||
|
||||
-- Manual/upload evidence stays at E1 + uploaded (default)
|
||||
|
||||
-- Generated evidence → E0 + generated
|
||||
UPDATE compliance_evidence
|
||||
SET confidence_level = 'E0',
|
||||
truth_status = 'generated',
|
||||
may_be_used_as_evidence = FALSE
|
||||
WHERE source = 'generated'
|
||||
AND confidence_level = 'E1';
|
||||
|
||||
-- ============================================================================
|
||||
-- 7. Indexes
|
||||
-- ============================================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_confidence ON compliance_evidence (confidence_level);
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_truth_status ON compliance_evidence (truth_status);
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_may_be_used ON compliance_evidence (may_be_used_as_evidence);
|
||||
CREATE INDEX IF NOT EXISTS ix_llm_audit_entity ON compliance_llm_generation_audit (entity_type, entity_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_llm_audit_tenant ON compliance_llm_generation_audit (tenant_id);
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Migration 077: Anti-Fake-Evidence Phase 2
|
||||
-- Assertions table, Four-Eyes columns on Evidence, Audit-Trail performance index
|
||||
|
||||
-- 1A. Assertions table
|
||||
CREATE TABLE IF NOT EXISTS compliance_assertions (
|
||||
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
||||
tenant_id VARCHAR(36),
|
||||
entity_type VARCHAR(50) NOT NULL,
|
||||
entity_id VARCHAR(36) NOT NULL,
|
||||
sentence_text TEXT NOT NULL,
|
||||
sentence_index INTEGER NOT NULL DEFAULT 0,
|
||||
assertion_type VARCHAR(20) NOT NULL DEFAULT 'assertion',
|
||||
evidence_ids JSONB DEFAULT '[]'::jsonb,
|
||||
confidence FLOAT DEFAULT 0.0,
|
||||
normative_tier VARCHAR(20),
|
||||
verified_by VARCHAR(200),
|
||||
verified_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_assertion_entity ON compliance_assertions (entity_type, entity_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_assertion_type ON compliance_assertions (assertion_type);
|
||||
CREATE INDEX IF NOT EXISTS ix_assertion_tenant ON compliance_assertions (tenant_id);
|
||||
|
||||
-- 1B. Four-Eyes columns on Evidence
|
||||
ALTER TABLE compliance_evidence
|
||||
ADD COLUMN IF NOT EXISTS approval_status VARCHAR(30) DEFAULT 'none',
|
||||
ADD COLUMN IF NOT EXISTS first_reviewer VARCHAR(200),
|
||||
ADD COLUMN IF NOT EXISTS first_reviewed_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS second_reviewer VARCHAR(200),
|
||||
ADD COLUMN IF NOT EXISTS second_reviewed_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS requires_four_eyes BOOLEAN DEFAULT FALSE;
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_approval_status ON compliance_evidence (approval_status);
|
||||
|
||||
-- 1C. Audit-Trail performance index
|
||||
CREATE INDEX IF NOT EXISTS ix_audit_trail_entity_action
|
||||
ON compliance_audit_trail (entity_type, action, performed_at);
|
||||
@@ -0,0 +1,562 @@
|
||||
"""Tests for Anti-Fake-Evidence Phase 1 guardrails.
|
||||
|
||||
~45 tests covering:
|
||||
- Evidence confidence classification
|
||||
- Evidence truth status classification
|
||||
- Control status transition state machine
|
||||
- Multi-dimensional compliance score
|
||||
- LLM generation audit
|
||||
- Evidence review endpoint
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from compliance.api.evidence_routes import router as evidence_router
|
||||
from compliance.api.llm_audit_routes import router as llm_audit_router
|
||||
from compliance.api.evidence_routes import _classify_confidence, _classify_truth_status
|
||||
from compliance.services.control_status_machine import validate_transition
|
||||
from compliance.db.models import (
|
||||
EvidenceConfidenceEnum,
|
||||
EvidenceTruthStatusEnum,
|
||||
ControlStatusEnum,
|
||||
)
|
||||
from classroom_engine.database import get_db
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# App setup with mocked DB dependency
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(evidence_router)
|
||||
app.include_router(llm_audit_router, prefix="/compliance")
|
||||
|
||||
mock_db = MagicMock()
|
||||
|
||||
|
||||
def override_get_db():
|
||||
yield mock_db
|
||||
|
||||
|
||||
app.dependency_overrides[get_db] = override_get_db
|
||||
client = TestClient(app)
|
||||
|
||||
EVIDENCE_UUID = "eeeeeeee-aaaa-bbbb-cccc-ffffffffffff"
|
||||
CONTROL_UUID = "cccccccc-aaaa-bbbb-cccc-dddddddddddd"
|
||||
NOW = datetime(2026, 3, 23, 12, 0, 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_evidence(overrides=None):
|
||||
e = MagicMock()
|
||||
e.id = EVIDENCE_UUID
|
||||
e.control_id = CONTROL_UUID
|
||||
e.evidence_type = "test_results"
|
||||
e.title = "Pytest Test Report"
|
||||
e.description = "All tests passing"
|
||||
e.artifact_url = "https://ci.example.com/job/123/artifact"
|
||||
e.artifact_path = None
|
||||
e.artifact_hash = "abc123def456"
|
||||
e.file_size_bytes = None
|
||||
e.mime_type = None
|
||||
e.status = MagicMock()
|
||||
e.status.value = "valid"
|
||||
e.uploaded_by = None
|
||||
e.source = "ci_pipeline"
|
||||
e.ci_job_id = "job-123"
|
||||
e.valid_from = NOW
|
||||
e.valid_until = NOW + timedelta(days=90)
|
||||
e.collected_at = NOW
|
||||
e.created_at = NOW
|
||||
# Anti-fake-evidence fields
|
||||
e.confidence_level = EvidenceConfidenceEnum.E3
|
||||
e.truth_status = EvidenceTruthStatusEnum.OBSERVED
|
||||
e.generation_mode = None
|
||||
e.may_be_used_as_evidence = True
|
||||
e.reviewed_by = None
|
||||
e.reviewed_at = None
|
||||
# Phase 2 fields
|
||||
e.approval_status = "none"
|
||||
e.first_reviewer = None
|
||||
e.first_reviewed_at = None
|
||||
e.second_reviewer = None
|
||||
e.second_reviewed_at = None
|
||||
e.requires_four_eyes = False
|
||||
if overrides:
|
||||
for k, v in overrides.items():
|
||||
setattr(e, k, v)
|
||||
return e
|
||||
|
||||
|
||||
def make_control(overrides=None):
|
||||
c = MagicMock()
|
||||
c.id = CONTROL_UUID
|
||||
c.control_id = "GOV-001"
|
||||
c.title = "Access Control"
|
||||
c.status = ControlStatusEnum.PLANNED
|
||||
if overrides:
|
||||
for k, v in overrides.items():
|
||||
setattr(c, k, v)
|
||||
return c
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 1. TestEvidenceConfidenceClassification
|
||||
# ===========================================================================
|
||||
|
||||
class TestEvidenceConfidenceClassification:
|
||||
"""Test automatic confidence level classification."""
|
||||
|
||||
def test_ci_pipeline_returns_e3(self):
|
||||
assert _classify_confidence("ci_pipeline") == EvidenceConfidenceEnum.E3
|
||||
|
||||
def test_api_with_hash_returns_e3(self):
|
||||
assert _classify_confidence("api", artifact_hash="sha256:abc") == EvidenceConfidenceEnum.E3
|
||||
|
||||
def test_api_without_hash_returns_e3(self):
|
||||
assert _classify_confidence("api") == EvidenceConfidenceEnum.E3
|
||||
|
||||
def test_manual_returns_e1(self):
|
||||
assert _classify_confidence("manual") == EvidenceConfidenceEnum.E1
|
||||
|
||||
def test_upload_returns_e1(self):
|
||||
assert _classify_confidence("upload") == EvidenceConfidenceEnum.E1
|
||||
|
||||
def test_generated_returns_e0(self):
|
||||
assert _classify_confidence("generated") == EvidenceConfidenceEnum.E0
|
||||
|
||||
def test_unknown_source_returns_e1(self):
|
||||
assert _classify_confidence("some_random_source") == EvidenceConfidenceEnum.E1
|
||||
|
||||
def test_none_source_returns_e1(self):
|
||||
assert _classify_confidence(None) == EvidenceConfidenceEnum.E1
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 2. TestEvidenceTruthStatus
|
||||
# ===========================================================================
|
||||
|
||||
class TestEvidenceTruthStatus:
|
||||
"""Test automatic truth status classification."""
|
||||
|
||||
def test_ci_pipeline_returns_observed(self):
|
||||
assert _classify_truth_status("ci_pipeline") == EvidenceTruthStatusEnum.OBSERVED
|
||||
|
||||
def test_manual_returns_uploaded(self):
|
||||
assert _classify_truth_status("manual") == EvidenceTruthStatusEnum.UPLOADED
|
||||
|
||||
def test_upload_returns_uploaded(self):
|
||||
assert _classify_truth_status("upload") == EvidenceTruthStatusEnum.UPLOADED
|
||||
|
||||
def test_generated_returns_generated(self):
|
||||
assert _classify_truth_status("generated") == EvidenceTruthStatusEnum.GENERATED
|
||||
|
||||
def test_api_returns_observed(self):
|
||||
assert _classify_truth_status("api") == EvidenceTruthStatusEnum.OBSERVED
|
||||
|
||||
def test_none_returns_uploaded(self):
|
||||
assert _classify_truth_status(None) == EvidenceTruthStatusEnum.UPLOADED
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 3. TestControlStatusTransitions
|
||||
# ===========================================================================
|
||||
|
||||
class TestControlStatusTransitions:
|
||||
"""Test the control status transition state machine."""
|
||||
|
||||
def test_planned_to_in_progress_allowed(self):
|
||||
allowed, violations = validate_transition("planned", "in_progress")
|
||||
assert allowed is True
|
||||
assert violations == []
|
||||
|
||||
def test_in_progress_to_pass_without_evidence_blocked(self):
|
||||
allowed, violations = validate_transition("in_progress", "pass", evidence_list=[])
|
||||
assert allowed is False
|
||||
assert len(violations) > 0
|
||||
assert "pass" in violations[0].lower()
|
||||
|
||||
def test_in_progress_to_pass_with_e2_evidence_allowed(self):
|
||||
e = make_evidence({
|
||||
"confidence_level": EvidenceConfidenceEnum.E2,
|
||||
"truth_status": EvidenceTruthStatusEnum.VALIDATED_INTERNAL,
|
||||
})
|
||||
allowed, violations = validate_transition("in_progress", "pass", evidence_list=[e])
|
||||
assert allowed is True
|
||||
assert violations == []
|
||||
|
||||
def test_in_progress_to_pass_with_e1_evidence_blocked(self):
|
||||
e = make_evidence({
|
||||
"confidence_level": EvidenceConfidenceEnum.E1,
|
||||
"truth_status": EvidenceTruthStatusEnum.UPLOADED,
|
||||
})
|
||||
allowed, violations = validate_transition("in_progress", "pass", evidence_list=[e])
|
||||
assert allowed is False
|
||||
assert "E2" in violations[0]
|
||||
|
||||
def test_in_progress_to_partial_with_evidence_allowed(self):
|
||||
e = make_evidence({"confidence_level": EvidenceConfidenceEnum.E0})
|
||||
allowed, violations = validate_transition("in_progress", "partial", evidence_list=[e])
|
||||
assert allowed is True
|
||||
|
||||
def test_in_progress_to_partial_without_evidence_blocked(self):
|
||||
allowed, violations = validate_transition("in_progress", "partial", evidence_list=[])
|
||||
assert allowed is False
|
||||
|
||||
def test_pass_to_fail_always_allowed(self):
|
||||
allowed, violations = validate_transition("pass", "fail")
|
||||
assert allowed is True
|
||||
|
||||
def test_any_to_na_requires_justification(self):
|
||||
allowed, violations = validate_transition("in_progress", "n/a", status_justification=None)
|
||||
assert allowed is False
|
||||
assert "justification" in violations[0].lower()
|
||||
|
||||
def test_any_to_na_with_justification_allowed(self):
|
||||
allowed, violations = validate_transition("in_progress", "n/a", status_justification="Not applicable for this project")
|
||||
assert allowed is True
|
||||
|
||||
def test_any_to_planned_always_allowed(self):
|
||||
allowed, violations = validate_transition("pass", "planned")
|
||||
assert allowed is True
|
||||
|
||||
def test_same_status_noop_allowed(self):
|
||||
allowed, violations = validate_transition("pass", "pass")
|
||||
assert allowed is True
|
||||
|
||||
def test_bypass_for_auto_updater(self):
|
||||
allowed, violations = validate_transition("in_progress", "pass", evidence_list=[], bypass_for_auto_updater=True)
|
||||
assert allowed is True
|
||||
|
||||
def test_partial_to_pass_needs_e2(self):
|
||||
e = make_evidence({
|
||||
"confidence_level": EvidenceConfidenceEnum.E1,
|
||||
"truth_status": EvidenceTruthStatusEnum.UPLOADED,
|
||||
})
|
||||
allowed, violations = validate_transition("partial", "pass", evidence_list=[e])
|
||||
assert allowed is False
|
||||
|
||||
def test_partial_to_pass_with_e3_allowed(self):
|
||||
e = make_evidence({
|
||||
"confidence_level": EvidenceConfidenceEnum.E3,
|
||||
"truth_status": EvidenceTruthStatusEnum.OBSERVED,
|
||||
})
|
||||
allowed, violations = validate_transition("partial", "pass", evidence_list=[e])
|
||||
assert allowed is True
|
||||
|
||||
def test_in_progress_to_fail_allowed(self):
|
||||
allowed, violations = validate_transition("in_progress", "fail")
|
||||
assert allowed is True
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 4. TestMultiDimensionalScore
|
||||
# ===========================================================================
|
||||
|
||||
class TestMultiDimensionalScore:
|
||||
"""Test multi-dimensional score calculation."""
|
||||
|
||||
def test_score_structure(self):
|
||||
"""Score result should have all required keys."""
|
||||
from compliance.db.repository import ControlRepository
|
||||
repo = ControlRepository(mock_db)
|
||||
|
||||
with patch.object(repo, 'get_all', return_value=[]):
|
||||
result = repo.get_multi_dimensional_score()
|
||||
|
||||
assert "requirement_coverage" in result
|
||||
assert "evidence_strength" in result
|
||||
assert "validation_quality" in result
|
||||
assert "evidence_freshness" in result
|
||||
assert "control_effectiveness" in result
|
||||
assert "overall_readiness" in result
|
||||
assert "hard_blocks" in result
|
||||
|
||||
def test_empty_controls_returns_zeros(self):
|
||||
from compliance.db.repository import ControlRepository
|
||||
repo = ControlRepository(mock_db)
|
||||
|
||||
with patch.object(repo, 'get_all', return_value=[]):
|
||||
result = repo.get_multi_dimensional_score()
|
||||
|
||||
assert result["overall_readiness"] == 0.0
|
||||
assert "Keine Controls" in result["hard_blocks"][0]
|
||||
|
||||
def test_hard_blocks_pass_without_evidence(self):
|
||||
"""Controls on 'pass' without evidence should trigger hard block."""
|
||||
from compliance.db.repository import ControlRepository
|
||||
repo = ControlRepository(mock_db)
|
||||
|
||||
ctrl = make_control({"status": ControlStatusEnum.PASS})
|
||||
mock_db.query.return_value.all.return_value = [] # no evidence
|
||||
mock_db.query.return_value.scalar.return_value = 0
|
||||
|
||||
with patch.object(repo, 'get_all', return_value=[ctrl]):
|
||||
result = repo.get_multi_dimensional_score()
|
||||
|
||||
assert any("Evidence" in b or "evidence" in b.lower() for b in result["hard_blocks"])
|
||||
|
||||
def test_all_dimensions_are_floats(self):
|
||||
from compliance.db.repository import ControlRepository
|
||||
repo = ControlRepository(mock_db)
|
||||
|
||||
with patch.object(repo, 'get_all', return_value=[]):
|
||||
result = repo.get_multi_dimensional_score()
|
||||
|
||||
for key in ["requirement_coverage", "evidence_strength", "validation_quality",
|
||||
"evidence_freshness", "control_effectiveness", "overall_readiness"]:
|
||||
assert isinstance(result[key], float), f"{key} should be float"
|
||||
|
||||
def test_hard_blocks_is_list(self):
|
||||
from compliance.db.repository import ControlRepository
|
||||
repo = ControlRepository(mock_db)
|
||||
|
||||
with patch.object(repo, 'get_all', return_value=[]):
|
||||
result = repo.get_multi_dimensional_score()
|
||||
|
||||
assert isinstance(result["hard_blocks"], list)
|
||||
|
||||
def test_backwards_compatibility_with_old_score(self):
|
||||
"""get_statistics should still work and return compliance_score."""
|
||||
from compliance.db.repository import ControlRepository
|
||||
repo = ControlRepository(mock_db)
|
||||
|
||||
mock_db.query.return_value.scalar.return_value = 0
|
||||
mock_db.query.return_value.group_by.return_value.all.return_value = []
|
||||
|
||||
result = repo.get_statistics()
|
||||
assert "compliance_score" in result
|
||||
assert "total" in result
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 5. TestForbiddenFormulations
|
||||
# ===========================================================================
|
||||
|
||||
class TestForbiddenFormulations:
|
||||
"""Test forbidden formulation detection (tested via the validate endpoint context)."""
|
||||
|
||||
def test_import_works(self):
|
||||
"""Verify forbidden pattern check function is importable and callable."""
|
||||
# This tests the Python-side schema, the actual check is in TypeScript
|
||||
from compliance.api.schemas import MultiDimensionalScore, StatusTransitionError
|
||||
score = MultiDimensionalScore()
|
||||
assert score.overall_readiness == 0.0
|
||||
err = StatusTransitionError(current_status="planned", requested_status="pass")
|
||||
assert err.allowed is False
|
||||
|
||||
def test_status_transition_error_schema(self):
|
||||
from compliance.api.schemas import StatusTransitionError
|
||||
err = StatusTransitionError(
|
||||
allowed=False,
|
||||
current_status="in_progress",
|
||||
requested_status="pass",
|
||||
violations=["Need E2 evidence"],
|
||||
)
|
||||
assert err.violations == ["Need E2 evidence"]
|
||||
|
||||
def test_multi_dimensional_score_defaults(self):
|
||||
from compliance.api.schemas import MultiDimensionalScore
|
||||
score = MultiDimensionalScore()
|
||||
assert score.requirement_coverage == 0.0
|
||||
assert score.hard_blocks == []
|
||||
|
||||
def test_multi_dimensional_score_with_data(self):
|
||||
from compliance.api.schemas import MultiDimensionalScore
|
||||
score = MultiDimensionalScore(
|
||||
requirement_coverage=80.0,
|
||||
evidence_strength=60.0,
|
||||
validation_quality=40.0,
|
||||
evidence_freshness=90.0,
|
||||
control_effectiveness=70.0,
|
||||
overall_readiness=65.0,
|
||||
hard_blocks=["3 Controls ohne Evidence"],
|
||||
)
|
||||
assert score.overall_readiness == 65.0
|
||||
assert len(score.hard_blocks) == 1
|
||||
|
||||
def test_evidence_response_has_anti_fake_fields(self):
|
||||
from compliance.api.schemas import EvidenceResponse
|
||||
fields = EvidenceResponse.model_fields
|
||||
assert "confidence_level" in fields
|
||||
assert "truth_status" in fields
|
||||
assert "generation_mode" in fields
|
||||
assert "may_be_used_as_evidence" in fields
|
||||
assert "reviewed_by" in fields
|
||||
assert "reviewed_at" in fields
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 6. TestLLMGenerationAudit
|
||||
# ===========================================================================
|
||||
|
||||
class TestLLMGenerationAudit:
|
||||
"""Test LLM generation audit trail."""
|
||||
|
||||
def test_create_audit_record(self):
|
||||
"""POST /compliance/llm-audit should create a record."""
|
||||
mock_record = MagicMock()
|
||||
mock_record.id = "audit-001"
|
||||
mock_record.tenant_id = None
|
||||
mock_record.entity_type = "document"
|
||||
mock_record.entity_id = None
|
||||
mock_record.generation_mode = "draft_assistance"
|
||||
mock_record.truth_status = EvidenceTruthStatusEnum.GENERATED
|
||||
mock_record.may_be_used_as_evidence = False
|
||||
mock_record.llm_model = "qwen2.5vl:32b"
|
||||
mock_record.llm_provider = "ollama"
|
||||
mock_record.prompt_hash = None
|
||||
mock_record.input_summary = "Test input"
|
||||
mock_record.output_summary = "Test output"
|
||||
mock_record.extra_metadata = {}
|
||||
mock_record.created_at = NOW
|
||||
|
||||
mock_db.add = MagicMock()
|
||||
mock_db.commit = MagicMock()
|
||||
mock_db.refresh = MagicMock(side_effect=lambda r: setattr(r, 'id', 'audit-001'))
|
||||
|
||||
# We need to patch the LLMGenerationAuditDB constructor
|
||||
with patch('compliance.api.llm_audit_routes.LLMGenerationAuditDB', return_value=mock_record):
|
||||
resp = client.post("/compliance/llm-audit", json={
|
||||
"entity_type": "document",
|
||||
"generation_mode": "draft_assistance",
|
||||
"truth_status": "generated",
|
||||
"may_be_used_as_evidence": False,
|
||||
"llm_model": "qwen2.5vl:32b",
|
||||
"llm_provider": "ollama",
|
||||
})
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["entity_type"] == "document"
|
||||
assert data["truth_status"] == "generated"
|
||||
assert data["may_be_used_as_evidence"] is False
|
||||
|
||||
def test_truth_status_always_generated_for_llm(self):
|
||||
"""LLM-generated content should always start with truth_status=generated."""
|
||||
from compliance.db.models import LLMGenerationAuditDB, EvidenceTruthStatusEnum
|
||||
audit = LLMGenerationAuditDB()
|
||||
# Default should be GENERATED
|
||||
assert audit.truth_status is None or audit.truth_status == EvidenceTruthStatusEnum.GENERATED
|
||||
|
||||
def test_may_be_used_as_evidence_defaults_false(self):
|
||||
"""Generated content should NOT be usable as evidence by default."""
|
||||
from compliance.db.models import LLMGenerationAuditDB
|
||||
audit = LLMGenerationAuditDB()
|
||||
assert audit.may_be_used_as_evidence is False or audit.may_be_used_as_evidence is None
|
||||
|
||||
def test_list_audit_records(self):
|
||||
"""GET /compliance/llm-audit should return records."""
|
||||
mock_query = MagicMock()
|
||||
mock_query.count.return_value = 0
|
||||
mock_query.filter.return_value = mock_query
|
||||
mock_query.order_by.return_value = mock_query
|
||||
mock_query.offset.return_value = mock_query
|
||||
mock_query.limit.return_value = mock_query
|
||||
mock_query.all.return_value = []
|
||||
mock_db.query.return_value = mock_query
|
||||
|
||||
resp = client.get("/compliance/llm-audit")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "records" in data
|
||||
assert "total" in data
|
||||
assert data["total"] == 0
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 7. TestEvidenceReview
|
||||
# ===========================================================================
|
||||
|
||||
class TestEvidenceReview:
|
||||
"""Test evidence review endpoint."""
|
||||
|
||||
def test_review_upgrades_confidence(self):
|
||||
"""PATCH /evidence/{id}/review should update confidence and set reviewer."""
|
||||
evidence = make_evidence({
|
||||
"confidence_level": EvidenceConfidenceEnum.E1,
|
||||
"truth_status": EvidenceTruthStatusEnum.UPLOADED,
|
||||
})
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.commit = MagicMock()
|
||||
mock_db.refresh = MagicMock()
|
||||
|
||||
resp = client.patch(f"/evidence/{EVIDENCE_UUID}/review", json={
|
||||
"confidence_level": "E2",
|
||||
"truth_status": "validated_internal",
|
||||
"reviewed_by": "auditor@example.com",
|
||||
})
|
||||
|
||||
assert resp.status_code == 200
|
||||
# Verify the evidence was updated
|
||||
assert evidence.confidence_level == EvidenceConfidenceEnum.E2
|
||||
assert evidence.truth_status == EvidenceTruthStatusEnum.VALIDATED_INTERNAL
|
||||
assert evidence.reviewed_by == "auditor@example.com"
|
||||
assert evidence.reviewed_at is not None
|
||||
|
||||
def test_review_nonexistent_evidence_returns_404(self):
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = None
|
||||
resp = client.patch("/evidence/nonexistent-id/review", json={
|
||||
"reviewed_by": "someone",
|
||||
})
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_review_invalid_confidence_returns_400(self):
|
||||
evidence = make_evidence()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
|
||||
resp = client.patch(f"/evidence/{EVIDENCE_UUID}/review", json={
|
||||
"confidence_level": "INVALID",
|
||||
"reviewed_by": "someone",
|
||||
})
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 8. TestControlUpdateIntegration
|
||||
# ===========================================================================
|
||||
|
||||
class TestControlUpdateIntegration:
|
||||
"""Test that ControlUpdate schema includes status_justification."""
|
||||
|
||||
def test_control_update_has_status_justification(self):
|
||||
from compliance.api.schemas import ControlUpdate
|
||||
fields = ControlUpdate.model_fields
|
||||
assert "status_justification" in fields
|
||||
|
||||
def test_control_response_has_status_justification(self):
|
||||
from compliance.api.schemas import ControlResponse
|
||||
fields = ControlResponse.model_fields
|
||||
assert "status_justification" in fields
|
||||
|
||||
def test_control_status_enum_has_in_progress(self):
|
||||
assert ControlStatusEnum.IN_PROGRESS.value == "in_progress"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 9. TestEvidenceEnums
|
||||
# ===========================================================================
|
||||
|
||||
class TestEvidenceEnums:
|
||||
"""Test the new evidence enums."""
|
||||
|
||||
def test_confidence_enum_values(self):
|
||||
assert EvidenceConfidenceEnum.E0.value == "E0"
|
||||
assert EvidenceConfidenceEnum.E1.value == "E1"
|
||||
assert EvidenceConfidenceEnum.E2.value == "E2"
|
||||
assert EvidenceConfidenceEnum.E3.value == "E3"
|
||||
assert EvidenceConfidenceEnum.E4.value == "E4"
|
||||
|
||||
def test_truth_status_enum_values(self):
|
||||
assert EvidenceTruthStatusEnum.GENERATED.value == "generated"
|
||||
assert EvidenceTruthStatusEnum.UPLOADED.value == "uploaded"
|
||||
assert EvidenceTruthStatusEnum.OBSERVED.value == "observed"
|
||||
assert EvidenceTruthStatusEnum.VALIDATED_INTERNAL.value == "validated_internal"
|
||||
assert EvidenceTruthStatusEnum.REJECTED.value == "rejected"
|
||||
assert EvidenceTruthStatusEnum.PROVIDED_TO_AUDITOR.value == "provided_to_auditor"
|
||||
assert EvidenceTruthStatusEnum.ACCEPTED_BY_AUDITOR.value == "accepted_by_auditor"
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Tests for Anti-Fake-Evidence Phase 2.
|
||||
|
||||
~35 tests covering:
|
||||
- Audit trail extension (evidence review/create logging)
|
||||
- Assertion engine (extraction, CRUD, verify, summary)
|
||||
- Four-Eyes review (domain check, first/second review, same-person reject)
|
||||
- UI badge data (response schema includes new fields)
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from compliance.api.evidence_routes import (
|
||||
router as evidence_router,
|
||||
_requires_four_eyes,
|
||||
_classify_confidence,
|
||||
_classify_truth_status,
|
||||
)
|
||||
from compliance.api.assertion_routes import router as assertion_router
|
||||
from compliance.services.assertion_engine import extract_assertions, _classify_sentence
|
||||
from compliance.db.models import (
|
||||
EvidenceConfidenceEnum,
|
||||
EvidenceTruthStatusEnum,
|
||||
ControlStatusEnum,
|
||||
AssertionDB,
|
||||
)
|
||||
from classroom_engine.database import get_db
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# App setup with mocked DB dependency
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(evidence_router)
|
||||
app.include_router(assertion_router)
|
||||
|
||||
mock_db = MagicMock()
|
||||
|
||||
|
||||
def override_get_db():
|
||||
yield mock_db
|
||||
|
||||
|
||||
app.dependency_overrides[get_db] = override_get_db
|
||||
client = TestClient(app)
|
||||
|
||||
EVIDENCE_UUID = "eeee0002-aaaa-bbbb-cccc-ffffffffffff"
|
||||
CONTROL_UUID = "cccc0002-aaaa-bbbb-cccc-dddddddddddd"
|
||||
ASSERTION_UUID = "aaaa0002-bbbb-cccc-dddd-eeeeeeeeeeee"
|
||||
NOW = datetime(2026, 3, 23, 14, 0, 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_evidence(overrides=None):
|
||||
e = MagicMock()
|
||||
e.id = EVIDENCE_UUID
|
||||
e.control_id = CONTROL_UUID
|
||||
e.evidence_type = "test_results"
|
||||
e.title = "Phase 2 Test Evidence"
|
||||
e.description = "Testing four-eyes"
|
||||
e.artifact_url = "https://ci.example.com/artifact"
|
||||
e.artifact_path = None
|
||||
e.artifact_hash = "abc123"
|
||||
e.file_size_bytes = None
|
||||
e.mime_type = None
|
||||
e.status = MagicMock()
|
||||
e.status.value = "valid"
|
||||
e.uploaded_by = None
|
||||
e.source = "api"
|
||||
e.ci_job_id = None
|
||||
e.valid_from = NOW
|
||||
e.valid_until = NOW + timedelta(days=90)
|
||||
e.collected_at = NOW
|
||||
e.created_at = NOW
|
||||
e.confidence_level = EvidenceConfidenceEnum.E1
|
||||
e.truth_status = EvidenceTruthStatusEnum.UPLOADED
|
||||
e.generation_mode = None
|
||||
e.may_be_used_as_evidence = True
|
||||
e.reviewed_by = None
|
||||
e.reviewed_at = None
|
||||
# Phase 2 fields
|
||||
e.approval_status = "none"
|
||||
e.first_reviewer = None
|
||||
e.first_reviewed_at = None
|
||||
e.second_reviewer = None
|
||||
e.second_reviewed_at = None
|
||||
e.requires_four_eyes = False
|
||||
if overrides:
|
||||
for k, v in overrides.items():
|
||||
setattr(e, k, v)
|
||||
return e
|
||||
|
||||
|
||||
def make_assertion(overrides=None):
|
||||
a = MagicMock()
|
||||
a.id = ASSERTION_UUID
|
||||
a.tenant_id = "tenant-001"
|
||||
a.entity_type = "control"
|
||||
a.entity_id = CONTROL_UUID
|
||||
a.sentence_text = "Test assertion sentence"
|
||||
a.sentence_index = 0
|
||||
a.assertion_type = "assertion"
|
||||
a.evidence_ids = []
|
||||
a.confidence = 0.0
|
||||
a.normative_tier = "pflicht"
|
||||
a.verified_by = None
|
||||
a.verified_at = None
|
||||
a.created_at = NOW
|
||||
a.updated_at = NOW
|
||||
if overrides:
|
||||
for k, v in overrides.items():
|
||||
setattr(a, k, v)
|
||||
return a
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 1. TestAuditTrailExtension
|
||||
# ===========================================================================
|
||||
|
||||
class TestAuditTrailExtension:
|
||||
"""Test that evidence review and create log audit trail entries."""
|
||||
|
||||
def test_review_evidence_logs_audit_trail(self):
|
||||
evidence = make_evidence()
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"confidence_level": "E2", "reviewed_by": "auditor@test.com"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
# db.add should be called for audit trail entries
|
||||
assert mock_db.add.called
|
||||
|
||||
def test_review_evidence_records_old_and_new_confidence(self):
|
||||
evidence = make_evidence({"confidence_level": EvidenceConfidenceEnum.E1})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"confidence_level": "E3", "reviewed_by": "reviewer@test.com"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_review_evidence_records_truth_status_change(self):
|
||||
evidence = make_evidence({"truth_status": EvidenceTruthStatusEnum.UPLOADED})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"truth_status": "validated_internal", "reviewed_by": "reviewer@test.com"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_review_nonexistent_evidence_returns_404(self):
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
"/evidence/nonexistent/review",
|
||||
json={"reviewed_by": "someone"},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_reject_evidence_logs_audit_trail(self):
|
||||
evidence = make_evidence()
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/reject",
|
||||
json={"reviewed_by": "auditor@test.com", "rejection_reason": "Fake evidence"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["approval_status"] == "rejected"
|
||||
|
||||
def test_reject_nonexistent_evidence_returns_404(self):
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
"/evidence/nonexistent/reject",
|
||||
json={"reviewed_by": "someone"},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_audit_trail_query_endpoint(self):
|
||||
mock_db.reset_mock()
|
||||
trail_entry = MagicMock()
|
||||
trail_entry.id = "trail-001"
|
||||
trail_entry.entity_type = "evidence"
|
||||
trail_entry.entity_id = EVIDENCE_UUID
|
||||
trail_entry.entity_name = "Test"
|
||||
trail_entry.action = "review"
|
||||
trail_entry.field_changed = "confidence_level"
|
||||
trail_entry.old_value = "E1"
|
||||
trail_entry.new_value = "E2"
|
||||
trail_entry.change_summary = None
|
||||
trail_entry.performed_by = "auditor"
|
||||
trail_entry.performed_at = NOW
|
||||
trail_entry.checksum = "abc"
|
||||
mock_db.query.return_value.filter.return_value.filter.return_value.order_by.return_value.limit.return_value.all.return_value = [trail_entry]
|
||||
|
||||
resp = client.get(f"/audit-trail?entity_type=evidence&entity_id={EVIDENCE_UUID}")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["total"] >= 1
|
||||
|
||||
def test_audit_trail_checksum_present(self):
|
||||
"""Audit trail entries should have a checksum for integrity."""
|
||||
from compliance.api.audit_trail_utils import create_signature
|
||||
sig = create_signature("evidence|123|review|user@test.com")
|
||||
assert len(sig) == 64 # SHA-256 hex digest
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 2. TestAssertionEngine
|
||||
# ===========================================================================
|
||||
|
||||
class TestAssertionEngine:
|
||||
"""Test assertion extraction and classification."""
|
||||
|
||||
def test_pflicht_sentence_classified_as_assertion(self):
|
||||
result = _classify_sentence("Die Organisation muss ein ISMS implementieren.")
|
||||
assert result == ("assertion", "pflicht")
|
||||
|
||||
def test_empfehlung_sentence_classified(self):
|
||||
result = _classify_sentence("Die Organisation sollte regelmäßige Audits durchführen.")
|
||||
assert result == ("assertion", "empfehlung")
|
||||
|
||||
def test_kann_sentence_classified(self):
|
||||
result = _classify_sentence("Optional kann ein externes Audit durchgeführt werden.")
|
||||
assert result == ("assertion", "kann")
|
||||
|
||||
def test_rationale_sentence_classified(self):
|
||||
result = _classify_sentence("Dies ist erforderlich, weil Datenverlust schwere Folgen hat.")
|
||||
assert result == ("rationale", None)
|
||||
|
||||
def test_fact_sentence_with_evidence_keyword(self):
|
||||
result = _classify_sentence("Das Zertifikat wurde am 15.03.2026 ausgestellt.")
|
||||
assert result == ("fact", None)
|
||||
|
||||
def test_extract_assertions_splits_sentences(self):
|
||||
text = "Die Organisation muss Daten schützen. Sie sollte regelmäßig prüfen."
|
||||
results = extract_assertions(text, "control", "ctrl-001")
|
||||
assert len(results) == 2
|
||||
assert results[0]["assertion_type"] == "assertion"
|
||||
assert results[0]["normative_tier"] == "pflicht"
|
||||
assert results[1]["normative_tier"] == "empfehlung"
|
||||
|
||||
def test_extract_assertions_empty_text(self):
|
||||
results = extract_assertions("", "control", "ctrl-001")
|
||||
assert results == []
|
||||
|
||||
def test_extract_assertions_single_sentence(self):
|
||||
results = extract_assertions("Der Betreiber muss ein Audit durchführen.", "control", "ctrl-001")
|
||||
assert len(results) == 1
|
||||
assert results[0]["normative_tier"] == "pflicht"
|
||||
|
||||
def test_mixed_text_with_rationale(self):
|
||||
text = "Die Organisation muss ein ISMS implementieren. Dies ist notwendig, weil Compliance gefordert ist."
|
||||
results = extract_assertions(text, "control", "ctrl-001")
|
||||
assert len(results) == 2
|
||||
types = [r["assertion_type"] for r in results]
|
||||
assert "assertion" in types
|
||||
assert "rationale" in types
|
||||
|
||||
def test_assertion_crud_create(self):
|
||||
mock_db.reset_mock()
|
||||
mock_db.refresh.return_value = None
|
||||
# Mock the added object to return proper values
|
||||
def side_effect_add(obj):
|
||||
obj.id = ASSERTION_UUID
|
||||
obj.created_at = NOW
|
||||
obj.updated_at = NOW
|
||||
obj.sentence_index = 0
|
||||
obj.confidence = 0.0
|
||||
mock_db.add.side_effect = side_effect_add
|
||||
|
||||
resp = client.post(
|
||||
"/assertions?tenant_id=tenant-001",
|
||||
json={
|
||||
"entity_type": "control",
|
||||
"entity_id": CONTROL_UUID,
|
||||
"sentence_text": "Die Organisation muss ein ISMS implementieren.",
|
||||
"assertion_type": "assertion",
|
||||
"normative_tier": "pflicht",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_assertion_verify_endpoint(self):
|
||||
a = make_assertion()
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = a
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.post(f"/assertions/{ASSERTION_UUID}/verify?verified_by=auditor@test.com")
|
||||
assert resp.status_code == 200
|
||||
assert a.assertion_type == "fact"
|
||||
assert a.verified_by == "auditor@test.com"
|
||||
|
||||
def test_assertion_summary(self):
|
||||
mock_db.reset_mock()
|
||||
a1 = make_assertion({"assertion_type": "assertion", "verified_by": None})
|
||||
a2 = make_assertion({"assertion_type": "fact", "verified_by": "user"})
|
||||
a3 = make_assertion({"assertion_type": "rationale", "verified_by": None})
|
||||
mock_db.query.return_value.filter.return_value.filter.return_value.filter.return_value.all.return_value = [a1, a2, a3]
|
||||
# Direct .all() for no-filter case
|
||||
mock_db.query.return_value.all.return_value = [a1, a2, a3]
|
||||
|
||||
resp = client.get("/assertions/summary")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["total_assertions"] == 3
|
||||
assert data["total_facts"] == 1
|
||||
assert data["total_rationale"] == 1
|
||||
assert data["unverified_count"] == 1
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 3. TestFourEyesReview
|
||||
# ===========================================================================
|
||||
|
||||
class TestFourEyesReview:
|
||||
"""Test Four-Eyes review process."""
|
||||
|
||||
def test_gov_domain_requires_four_eyes(self):
|
||||
assert _requires_four_eyes("gov") is True
|
||||
|
||||
def test_priv_domain_requires_four_eyes(self):
|
||||
assert _requires_four_eyes("priv") is True
|
||||
|
||||
def test_ops_domain_does_not_require_four_eyes(self):
|
||||
assert _requires_four_eyes("ops") is False
|
||||
|
||||
def test_sdlc_domain_does_not_require_four_eyes(self):
|
||||
assert _requires_four_eyes("sdlc") is False
|
||||
|
||||
def test_first_review_sets_first_approved(self):
|
||||
evidence = make_evidence({
|
||||
"requires_four_eyes": True,
|
||||
"approval_status": "pending_first",
|
||||
})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"reviewed_by": "reviewer1@test.com"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert evidence.first_reviewer == "reviewer1@test.com"
|
||||
assert evidence.approval_status == "first_approved"
|
||||
|
||||
def test_second_review_different_person_approves(self):
|
||||
evidence = make_evidence({
|
||||
"requires_four_eyes": True,
|
||||
"approval_status": "first_approved",
|
||||
"first_reviewer": "reviewer1@test.com",
|
||||
})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"reviewed_by": "reviewer2@test.com"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert evidence.second_reviewer == "reviewer2@test.com"
|
||||
assert evidence.approval_status == "approved"
|
||||
|
||||
def test_same_person_second_review_rejected(self):
|
||||
evidence = make_evidence({
|
||||
"requires_four_eyes": True,
|
||||
"approval_status": "first_approved",
|
||||
"first_reviewer": "reviewer1@test.com",
|
||||
})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"reviewed_by": "reviewer1@test.com"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "different" in resp.json()["detail"].lower()
|
||||
|
||||
def test_already_approved_blocked(self):
|
||||
evidence = make_evidence({
|
||||
"requires_four_eyes": True,
|
||||
"approval_status": "approved",
|
||||
})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"reviewed_by": "reviewer3@test.com"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
assert "already" in resp.json()["detail"].lower()
|
||||
|
||||
def test_rejected_evidence_cannot_be_reviewed(self):
|
||||
evidence = make_evidence({
|
||||
"requires_four_eyes": True,
|
||||
"approval_status": "rejected",
|
||||
})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"reviewed_by": "reviewer@test.com"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_reject_endpoint(self):
|
||||
evidence = make_evidence({"requires_four_eyes": True})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/reject",
|
||||
json={"reviewed_by": "auditor@test.com", "rejection_reason": "Not authentic"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert evidence.approval_status == "rejected"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 4. TestUIBadgeData
|
||||
# ===========================================================================
|
||||
|
||||
class TestUIBadgeData:
|
||||
"""Test that evidence response includes all Phase 2 fields."""
|
||||
|
||||
def test_evidence_response_includes_approval_status(self):
|
||||
evidence = make_evidence({
|
||||
"approval_status": "first_approved",
|
||||
"first_reviewer": "reviewer1@test.com",
|
||||
"first_reviewed_at": NOW,
|
||||
"requires_four_eyes": True,
|
||||
})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
mock_db.refresh.return_value = None
|
||||
|
||||
resp = client.patch(
|
||||
f"/evidence/{EVIDENCE_UUID}/review",
|
||||
json={"reviewed_by": "reviewer2@test.com"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "approval_status" in data
|
||||
assert "requires_four_eyes" in data
|
||||
assert data["requires_four_eyes"] is True
|
||||
|
||||
def test_evidence_response_includes_four_eyes_fields(self):
|
||||
evidence = make_evidence({
|
||||
"requires_four_eyes": True,
|
||||
"approval_status": "approved",
|
||||
"first_reviewer": "r1@test.com",
|
||||
"first_reviewed_at": NOW,
|
||||
"second_reviewer": "r2@test.com",
|
||||
"second_reviewed_at": NOW,
|
||||
})
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = evidence
|
||||
|
||||
# Use list endpoint
|
||||
mock_db.query.return_value.filter.return_value.all.return_value = [evidence]
|
||||
mock_db.query.return_value.all.return_value = [evidence]
|
||||
|
||||
# Direct test via _build_evidence_response
|
||||
from compliance.api.evidence_routes import _build_evidence_response
|
||||
resp = _build_evidence_response(evidence)
|
||||
assert resp.approval_status == "approved"
|
||||
assert resp.first_reviewer == "r1@test.com"
|
||||
assert resp.second_reviewer == "r2@test.com"
|
||||
assert resp.requires_four_eyes is True
|
||||
|
||||
def test_assertion_response_schema(self):
|
||||
a = make_assertion()
|
||||
mock_db.reset_mock()
|
||||
mock_db.query.return_value.filter.return_value.first.return_value = a
|
||||
|
||||
resp = client.get(f"/assertions/{ASSERTION_UUID}")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "assertion_type" in data
|
||||
assert "normative_tier" in data
|
||||
assert "evidence_ids" in data
|
||||
assert "verified_by" in data
|
||||
|
||||
def test_evidence_response_includes_confidence_and_truth(self):
|
||||
evidence = make_evidence({
|
||||
"confidence_level": EvidenceConfidenceEnum.E3,
|
||||
"truth_status": EvidenceTruthStatusEnum.OBSERVED,
|
||||
})
|
||||
from compliance.api.evidence_routes import _build_evidence_response
|
||||
resp = _build_evidence_response(evidence)
|
||||
assert resp.confidence_level == "E3"
|
||||
assert resp.truth_status == "observed"
|
||||
|
||||
def test_evidence_response_none_four_eyes_fields_default(self):
|
||||
evidence = make_evidence()
|
||||
from compliance.api.evidence_routes import _build_evidence_response
|
||||
resp = _build_evidence_response(evidence)
|
||||
assert resp.approval_status == "none"
|
||||
assert resp.requires_four_eyes is False
|
||||
assert resp.first_reviewer is None
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Tests for Anti-Fake-Evidence Phase 3: Enforcement.
|
||||
|
||||
~8 tests covering:
|
||||
- Evidence distribution endpoint (confidence counts, four-eyes pending)
|
||||
- Dashboard multi-score presence
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch, PropertyMock
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from compliance.api.dashboard_routes import router as dashboard_router
|
||||
from compliance.db.models import EvidenceConfidenceEnum, EvidenceTruthStatusEnum
|
||||
from classroom_engine.database import get_db
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# App setup with mocked DB dependency
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(dashboard_router)
|
||||
|
||||
mock_db = MagicMock()
|
||||
|
||||
|
||||
def override_get_db():
|
||||
yield mock_db
|
||||
|
||||
|
||||
app.dependency_overrides[get_db] = override_get_db
|
||||
client = TestClient(app)
|
||||
|
||||
NOW = datetime(2026, 3, 23, 14, 0, 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_evidence(confidence="E1", requires_four_eyes=False, approval_status="none"):
|
||||
e = MagicMock()
|
||||
e.confidence_level = MagicMock()
|
||||
e.confidence_level.value = confidence
|
||||
e.requires_four_eyes = requires_four_eyes
|
||||
e.approval_status = approval_status
|
||||
return e
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 1. TestEvidenceDistributionEndpoint
|
||||
# ===========================================================================
|
||||
|
||||
class TestEvidenceDistributionEndpoint:
|
||||
"""Test GET /dashboard/evidence-distribution endpoint."""
|
||||
|
||||
def _setup_evidence(self, evidence_list):
|
||||
"""Configure mock DB to return evidence list via EvidenceRepository."""
|
||||
mock_db.reset_mock()
|
||||
# EvidenceRepository(db).get_all() internally does db.query(...).all()
|
||||
# We patch the EvidenceRepository class to return our list
|
||||
return evidence_list
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
def test_empty_db_returns_zero_counts(self, mock_repo_cls):
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_all.return_value = []
|
||||
mock_repo_cls.return_value = mock_repo
|
||||
|
||||
resp = client.get("/dashboard/evidence-distribution")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["total"] == 0
|
||||
assert data["four_eyes_pending"] == 0
|
||||
assert data["by_confidence"] == {"E0": 0, "E1": 0, "E2": 0, "E3": 0, "E4": 0}
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
def test_counts_by_confidence_level(self, mock_repo_cls):
|
||||
evidence = [
|
||||
make_evidence("E0"),
|
||||
make_evidence("E1"),
|
||||
make_evidence("E1"),
|
||||
make_evidence("E2"),
|
||||
make_evidence("E3"),
|
||||
make_evidence("E3"),
|
||||
make_evidence("E3"),
|
||||
make_evidence("E4"),
|
||||
]
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_all.return_value = evidence
|
||||
mock_repo_cls.return_value = mock_repo
|
||||
|
||||
resp = client.get("/dashboard/evidence-distribution")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["total"] == 8
|
||||
assert data["by_confidence"]["E0"] == 1
|
||||
assert data["by_confidence"]["E1"] == 2
|
||||
assert data["by_confidence"]["E2"] == 1
|
||||
assert data["by_confidence"]["E3"] == 3
|
||||
assert data["by_confidence"]["E4"] == 1
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
def test_four_eyes_pending_count(self, mock_repo_cls):
|
||||
evidence = [
|
||||
make_evidence("E1", requires_four_eyes=True, approval_status="pending_first"),
|
||||
make_evidence("E2", requires_four_eyes=True, approval_status="first_approved"),
|
||||
make_evidence("E2", requires_four_eyes=True, approval_status="approved"),
|
||||
make_evidence("E1", requires_four_eyes=True, approval_status="rejected"),
|
||||
make_evidence("E1", requires_four_eyes=False, approval_status="none"),
|
||||
]
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_all.return_value = evidence
|
||||
mock_repo_cls.return_value = mock_repo
|
||||
|
||||
resp = client.get("/dashboard/evidence-distribution")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
# pending_first and first_approved are pending; approved and rejected are not
|
||||
assert data["four_eyes_pending"] == 2
|
||||
assert data["total"] == 5
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
def test_null_confidence_defaults_to_e1(self, mock_repo_cls):
|
||||
e = MagicMock()
|
||||
e.confidence_level = None
|
||||
e.requires_four_eyes = False
|
||||
e.approval_status = "none"
|
||||
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_all.return_value = [e]
|
||||
mock_repo_cls.return_value = mock_repo
|
||||
|
||||
resp = client.get("/dashboard/evidence-distribution")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["by_confidence"]["E1"] == 1
|
||||
assert data["total"] == 1
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
def test_all_four_eyes_approved_zero_pending(self, mock_repo_cls):
|
||||
evidence = [
|
||||
make_evidence("E2", requires_four_eyes=True, approval_status="approved"),
|
||||
make_evidence("E3", requires_four_eyes=True, approval_status="approved"),
|
||||
]
|
||||
mock_repo = MagicMock()
|
||||
mock_repo.get_all.return_value = evidence
|
||||
mock_repo_cls.return_value = mock_repo
|
||||
|
||||
resp = client.get("/dashboard/evidence-distribution")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["four_eyes_pending"] == 0
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 2. TestDashboardMultiScore
|
||||
# ===========================================================================
|
||||
|
||||
class TestDashboardMultiScore:
|
||||
"""Test that dashboard response includes multi_score."""
|
||||
|
||||
def test_dashboard_response_schema_includes_multi_score(self):
|
||||
"""DashboardResponse schema must include the multi_score field."""
|
||||
from compliance.api.schemas import DashboardResponse
|
||||
fields = DashboardResponse.model_fields
|
||||
assert "multi_score" in fields, "DashboardResponse must have multi_score field"
|
||||
|
||||
def test_multi_score_schema_has_required_fields(self):
|
||||
"""MultiDimensionalScore schema should have all 7 fields."""
|
||||
from compliance.api.schemas import MultiDimensionalScore
|
||||
fields = MultiDimensionalScore.model_fields
|
||||
required = [
|
||||
"requirement_coverage",
|
||||
"evidence_strength",
|
||||
"validation_quality",
|
||||
"evidence_freshness",
|
||||
"control_effectiveness",
|
||||
"overall_readiness",
|
||||
"hard_blocks",
|
||||
]
|
||||
for field in required:
|
||||
assert field in fields, f"Missing field: {field}"
|
||||
|
||||
def test_multi_score_default_values(self):
|
||||
"""MultiDimensionalScore defaults should be sensible."""
|
||||
from compliance.api.schemas import MultiDimensionalScore
|
||||
score = MultiDimensionalScore()
|
||||
assert score.overall_readiness == 0.0
|
||||
assert score.hard_blocks == []
|
||||
assert score.requirement_coverage == 0.0
|
||||
@@ -0,0 +1,277 @@
|
||||
"""Tests for Anti-Fake-Evidence Phase 4a: Traceability Matrix.
|
||||
|
||||
6 tests covering:
|
||||
- Empty DB returns empty controls + zero summary
|
||||
- Nested structure: Control → Evidence → Assertions
|
||||
- Assertions appear under correct evidence
|
||||
- Coverage flags computed correctly
|
||||
- Control without evidence has correct coverage
|
||||
- Summary counts match
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from compliance.api.dashboard_routes import router as dashboard_router
|
||||
from classroom_engine.database import get_db
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# App setup with mocked DB dependency
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(dashboard_router)
|
||||
|
||||
mock_db = MagicMock()
|
||||
|
||||
|
||||
def override_get_db():
|
||||
yield mock_db
|
||||
|
||||
|
||||
app.dependency_overrides[get_db] = override_get_db
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def make_control(id="c1", control_id="CTRL-001", title="Test Control", status="pass", domain="gov"):
|
||||
ctrl = MagicMock()
|
||||
ctrl.id = id
|
||||
ctrl.control_id = control_id
|
||||
ctrl.title = title
|
||||
ctrl.status = MagicMock()
|
||||
ctrl.status.value = status
|
||||
ctrl.domain = MagicMock()
|
||||
ctrl.domain.value = domain
|
||||
return ctrl
|
||||
|
||||
|
||||
def make_evidence(id="e1", control_id="c1", title="Evidence 1", evidence_type="scan_report",
|
||||
confidence="E2", status="valid"):
|
||||
e = MagicMock()
|
||||
e.id = id
|
||||
e.control_id = control_id
|
||||
e.title = title
|
||||
e.evidence_type = evidence_type
|
||||
e.confidence_level = MagicMock()
|
||||
e.confidence_level.value = confidence
|
||||
e.status = MagicMock()
|
||||
e.status.value = status
|
||||
return e
|
||||
|
||||
|
||||
def make_assertion(id="a1", entity_id="e1", sentence_text="System encrypts data at rest.",
|
||||
assertion_type="assertion", confidence=0.85, verified_by=None):
|
||||
a = MagicMock()
|
||||
a.id = id
|
||||
a.entity_id = entity_id
|
||||
a.sentence_text = sentence_text
|
||||
a.assertion_type = assertion_type
|
||||
a.confidence = confidence
|
||||
a.verified_by = verified_by
|
||||
return a
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Tests
|
||||
# ===========================================================================
|
||||
|
||||
class TestTraceabilityMatrix:
|
||||
"""Test GET /dashboard/traceability-matrix endpoint."""
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
@patch("compliance.api.dashboard_routes.ControlRepository")
|
||||
def test_empty_db_returns_empty_matrix(self, mock_ctrl_cls, mock_ev_cls):
|
||||
"""Empty DB should return zero controls and zero summary counts."""
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_all.return_value = []
|
||||
mock_ctrl_cls.return_value = mock_ctrl
|
||||
|
||||
mock_ev = MagicMock()
|
||||
mock_ev.get_all.return_value = []
|
||||
mock_ev_cls.return_value = mock_ev
|
||||
|
||||
# Mock db.query(AssertionDB).filter(...).all()
|
||||
mock_db.reset_mock()
|
||||
mock_query = MagicMock()
|
||||
mock_query.filter.return_value.all.return_value = []
|
||||
mock_db.query.return_value = mock_query
|
||||
|
||||
resp = client.get("/dashboard/traceability-matrix")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["controls"] == []
|
||||
assert data["summary"]["total_controls"] == 0
|
||||
assert data["summary"]["covered_controls"] == 0
|
||||
assert data["summary"]["fully_verified"] == 0
|
||||
assert data["summary"]["uncovered_controls"] == 0
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
@patch("compliance.api.dashboard_routes.ControlRepository")
|
||||
def test_nested_structure(self, mock_ctrl_cls, mock_ev_cls):
|
||||
"""Control with evidence and assertions should return nested structure."""
|
||||
ctrl = make_control(id="c1", control_id="PRIV-001", title="Privacy Control")
|
||||
ev = make_evidence(id="e1", control_id="c1", confidence="E3")
|
||||
assertion = make_assertion(id="a1", entity_id="e1", verified_by="auditor@example.com")
|
||||
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_all.return_value = [ctrl]
|
||||
mock_ctrl_cls.return_value = mock_ctrl
|
||||
|
||||
mock_ev = MagicMock()
|
||||
mock_ev.get_all.return_value = [ev]
|
||||
mock_ev_cls.return_value = mock_ev
|
||||
|
||||
mock_db.reset_mock()
|
||||
mock_query = MagicMock()
|
||||
mock_query.filter.return_value.all.return_value = [assertion]
|
||||
mock_db.query.return_value = mock_query
|
||||
|
||||
resp = client.get("/dashboard/traceability-matrix")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
|
||||
assert len(data["controls"]) == 1
|
||||
c = data["controls"][0]
|
||||
assert c["control_id"] == "PRIV-001"
|
||||
assert len(c["evidence"]) == 1
|
||||
assert c["evidence"][0]["confidence_level"] == "E3"
|
||||
assert len(c["evidence"][0]["assertions"]) == 1
|
||||
assert c["evidence"][0]["assertions"][0]["verified"] is True
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
@patch("compliance.api.dashboard_routes.ControlRepository")
|
||||
def test_assertions_grouped_under_correct_evidence(self, mock_ctrl_cls, mock_ev_cls):
|
||||
"""Assertions should only appear under the evidence they reference."""
|
||||
ctrl = make_control(id="c1")
|
||||
ev1 = make_evidence(id="e1", control_id="c1", title="Evidence A")
|
||||
ev2 = make_evidence(id="e2", control_id="c1", title="Evidence B")
|
||||
a1 = make_assertion(id="a1", entity_id="e1", sentence_text="Assertion for E1")
|
||||
a2 = make_assertion(id="a2", entity_id="e2", sentence_text="Assertion for E2")
|
||||
a3 = make_assertion(id="a3", entity_id="e2", sentence_text="Second assertion for E2")
|
||||
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_all.return_value = [ctrl]
|
||||
mock_ctrl_cls.return_value = mock_ctrl
|
||||
|
||||
mock_ev = MagicMock()
|
||||
mock_ev.get_all.return_value = [ev1, ev2]
|
||||
mock_ev_cls.return_value = mock_ev
|
||||
|
||||
mock_db.reset_mock()
|
||||
mock_query = MagicMock()
|
||||
mock_query.filter.return_value.all.return_value = [a1, a2, a3]
|
||||
mock_db.query.return_value = mock_query
|
||||
|
||||
resp = client.get("/dashboard/traceability-matrix")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
|
||||
c = data["controls"][0]
|
||||
ev1_data = next(e for e in c["evidence"] if e["id"] == "e1")
|
||||
ev2_data = next(e for e in c["evidence"] if e["id"] == "e2")
|
||||
assert len(ev1_data["assertions"]) == 1
|
||||
assert len(ev2_data["assertions"]) == 2
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
@patch("compliance.api.dashboard_routes.ControlRepository")
|
||||
def test_coverage_flags_correct(self, mock_ctrl_cls, mock_ev_cls):
|
||||
"""Coverage flags should reflect evidence, assertions, and verification state."""
|
||||
ctrl = make_control(id="c1")
|
||||
ev = make_evidence(id="e1", control_id="c1", confidence="E2")
|
||||
# One verified, one not
|
||||
a1 = make_assertion(id="a1", entity_id="e1", verified_by="alice")
|
||||
a2 = make_assertion(id="a2", entity_id="e1", verified_by=None)
|
||||
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_all.return_value = [ctrl]
|
||||
mock_ctrl_cls.return_value = mock_ctrl
|
||||
|
||||
mock_ev = MagicMock()
|
||||
mock_ev.get_all.return_value = [ev]
|
||||
mock_ev_cls.return_value = mock_ev
|
||||
|
||||
mock_db.reset_mock()
|
||||
mock_query = MagicMock()
|
||||
mock_query.filter.return_value.all.return_value = [a1, a2]
|
||||
mock_db.query.return_value = mock_query
|
||||
|
||||
resp = client.get("/dashboard/traceability-matrix")
|
||||
assert resp.status_code == 200
|
||||
|
||||
cov = resp.json()["controls"][0]["coverage"]
|
||||
assert cov["has_evidence"] is True
|
||||
assert cov["has_assertions"] is True
|
||||
assert cov["all_assertions_verified"] is False # a2 not verified
|
||||
assert cov["min_confidence_level"] == "E2"
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
@patch("compliance.api.dashboard_routes.ControlRepository")
|
||||
def test_coverage_without_evidence(self, mock_ctrl_cls, mock_ev_cls):
|
||||
"""Control with no evidence should have all coverage flags False/None."""
|
||||
ctrl = make_control(id="c1")
|
||||
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_all.return_value = [ctrl]
|
||||
mock_ctrl_cls.return_value = mock_ctrl
|
||||
|
||||
mock_ev = MagicMock()
|
||||
mock_ev.get_all.return_value = []
|
||||
mock_ev_cls.return_value = mock_ev
|
||||
|
||||
mock_db.reset_mock()
|
||||
mock_query = MagicMock()
|
||||
mock_query.filter.return_value.all.return_value = []
|
||||
mock_db.query.return_value = mock_query
|
||||
|
||||
resp = client.get("/dashboard/traceability-matrix")
|
||||
assert resp.status_code == 200
|
||||
|
||||
cov = resp.json()["controls"][0]["coverage"]
|
||||
assert cov["has_evidence"] is False
|
||||
assert cov["has_assertions"] is False
|
||||
assert cov["all_assertions_verified"] is False
|
||||
assert cov["min_confidence_level"] is None
|
||||
|
||||
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
||||
@patch("compliance.api.dashboard_routes.ControlRepository")
|
||||
def test_summary_counts(self, mock_ctrl_cls, mock_ev_cls):
|
||||
"""Summary should count total, covered, fully verified, and uncovered controls."""
|
||||
# c1: has evidence + verified assertions → fully verified
|
||||
# c2: has evidence but no assertions → covered, not fully verified
|
||||
# c3: no evidence → uncovered
|
||||
c1 = make_control(id="c1", control_id="C-001")
|
||||
c2 = make_control(id="c2", control_id="C-002")
|
||||
c3 = make_control(id="c3", control_id="C-003")
|
||||
|
||||
ev1 = make_evidence(id="e1", control_id="c1", confidence="E3")
|
||||
ev2 = make_evidence(id="e2", control_id="c2", confidence="E1")
|
||||
|
||||
a1 = make_assertion(id="a1", entity_id="e1", verified_by="auditor")
|
||||
|
||||
mock_ctrl = MagicMock()
|
||||
mock_ctrl.get_all.return_value = [c1, c2, c3]
|
||||
mock_ctrl_cls.return_value = mock_ctrl
|
||||
|
||||
mock_ev = MagicMock()
|
||||
mock_ev.get_all.return_value = [ev1, ev2]
|
||||
mock_ev_cls.return_value = mock_ev
|
||||
|
||||
mock_db.reset_mock()
|
||||
mock_query = MagicMock()
|
||||
mock_query.filter.return_value.all.return_value = [a1]
|
||||
mock_db.query.return_value = mock_query
|
||||
|
||||
resp = client.get("/dashboard/traceability-matrix")
|
||||
assert resp.status_code == 200
|
||||
|
||||
summary = resp.json()["summary"]
|
||||
assert summary["total_controls"] == 3
|
||||
assert summary["covered_controls"] == 2
|
||||
assert summary["fully_verified"] == 1
|
||||
assert summary["uncovered_controls"] == 1
|
||||
@@ -61,6 +61,7 @@ def make_control(overrides=None):
|
||||
c.status = MagicMock()
|
||||
c.status.value = "planned"
|
||||
c.status_notes = None
|
||||
c.status_justification = None
|
||||
c.last_reviewed_at = None
|
||||
c.next_review_at = None
|
||||
c.created_at = NOW
|
||||
@@ -249,15 +250,15 @@ class TestUpdateControl:
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_update_status_with_valid_enum(self):
|
||||
"""Status must be a valid ControlStatusEnum value."""
|
||||
"""Status must be a valid ControlStatusEnum value (planned → in_progress is always allowed)."""
|
||||
updated = make_control()
|
||||
updated.status.value = "pass"
|
||||
updated.status.value = "in_progress"
|
||||
with patch("compliance.api.routes.ControlRepository") as MockRepo:
|
||||
MockRepo.return_value.get_by_control_id.return_value = make_control()
|
||||
MockRepo.return_value.update.return_value = updated
|
||||
response = client.put(
|
||||
"/compliance/controls/GOV-001",
|
||||
json={"status": "pass"},
|
||||
json={"status": "in_progress"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@@ -56,6 +56,22 @@ def make_evidence(overrides=None):
|
||||
e.valid_until = None
|
||||
e.collected_at = NOW
|
||||
e.created_at = NOW
|
||||
# Anti-Fake-Evidence fields
|
||||
e.confidence_level = MagicMock()
|
||||
e.confidence_level.value = "E1"
|
||||
e.truth_status = MagicMock()
|
||||
e.truth_status.value = "uploaded"
|
||||
e.generation_mode = None
|
||||
e.may_be_used_as_evidence = True
|
||||
e.reviewed_by = None
|
||||
e.reviewed_at = None
|
||||
# Phase 2 fields
|
||||
e.approval_status = "none"
|
||||
e.first_reviewer = None
|
||||
e.first_reviewed_at = None
|
||||
e.second_reviewer = None
|
||||
e.second_reviewed_at = None
|
||||
e.requires_four_eyes = False
|
||||
if overrides:
|
||||
for k, v in overrides.items():
|
||||
setattr(e, k, v)
|
||||
|
||||
Reference in New Issue
Block a user