e6201d5239
Implement full evidence integrity pipeline to prevent compliance theater: - Confidence levels (E0-E4), truth status tracking, assertion engine - Four-Eyes approval workflow, audit trail, reject endpoint - Evidence distribution dashboard, LLM audit routes - Traceability matrix (backend endpoint + Compliance Hub UI tab) - Anti-fake badges, control status machine, normative patterns - 2 migrations, 4 test suites, MkDocs documentation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
278 lines
10 KiB
Python
278 lines
10 KiB
Python
"""Tests for Anti-Fake-Evidence Phase 4a: Traceability Matrix.
|
|
|
|
6 tests covering:
|
|
- Empty DB returns empty controls + zero summary
|
|
- Nested structure: Control → Evidence → Assertions
|
|
- Assertions appear under correct evidence
|
|
- Coverage flags computed correctly
|
|
- Control without evidence has correct coverage
|
|
- Summary counts match
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from unittest.mock import MagicMock, patch
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from compliance.api.dashboard_routes import router as dashboard_router
|
|
from classroom_engine.database import get_db
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# App setup with mocked DB dependency
|
|
# ---------------------------------------------------------------------------
|
|
|
|
app = FastAPI()
|
|
app.include_router(dashboard_router)
|
|
|
|
mock_db = MagicMock()
|
|
|
|
|
|
def override_get_db():
|
|
yield mock_db
|
|
|
|
|
|
app.dependency_overrides[get_db] = override_get_db
|
|
client = TestClient(app)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def make_control(id="c1", control_id="CTRL-001", title="Test Control", status="pass", domain="gov"):
|
|
ctrl = MagicMock()
|
|
ctrl.id = id
|
|
ctrl.control_id = control_id
|
|
ctrl.title = title
|
|
ctrl.status = MagicMock()
|
|
ctrl.status.value = status
|
|
ctrl.domain = MagicMock()
|
|
ctrl.domain.value = domain
|
|
return ctrl
|
|
|
|
|
|
def make_evidence(id="e1", control_id="c1", title="Evidence 1", evidence_type="scan_report",
|
|
confidence="E2", status="valid"):
|
|
e = MagicMock()
|
|
e.id = id
|
|
e.control_id = control_id
|
|
e.title = title
|
|
e.evidence_type = evidence_type
|
|
e.confidence_level = MagicMock()
|
|
e.confidence_level.value = confidence
|
|
e.status = MagicMock()
|
|
e.status.value = status
|
|
return e
|
|
|
|
|
|
def make_assertion(id="a1", entity_id="e1", sentence_text="System encrypts data at rest.",
|
|
assertion_type="assertion", confidence=0.85, verified_by=None):
|
|
a = MagicMock()
|
|
a.id = id
|
|
a.entity_id = entity_id
|
|
a.sentence_text = sentence_text
|
|
a.assertion_type = assertion_type
|
|
a.confidence = confidence
|
|
a.verified_by = verified_by
|
|
return a
|
|
|
|
|
|
# ===========================================================================
|
|
# Tests
|
|
# ===========================================================================
|
|
|
|
class TestTraceabilityMatrix:
|
|
"""Test GET /dashboard/traceability-matrix endpoint."""
|
|
|
|
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
|
@patch("compliance.api.dashboard_routes.ControlRepository")
|
|
def test_empty_db_returns_empty_matrix(self, mock_ctrl_cls, mock_ev_cls):
|
|
"""Empty DB should return zero controls and zero summary counts."""
|
|
mock_ctrl = MagicMock()
|
|
mock_ctrl.get_all.return_value = []
|
|
mock_ctrl_cls.return_value = mock_ctrl
|
|
|
|
mock_ev = MagicMock()
|
|
mock_ev.get_all.return_value = []
|
|
mock_ev_cls.return_value = mock_ev
|
|
|
|
# Mock db.query(AssertionDB).filter(...).all()
|
|
mock_db.reset_mock()
|
|
mock_query = MagicMock()
|
|
mock_query.filter.return_value.all.return_value = []
|
|
mock_db.query.return_value = mock_query
|
|
|
|
resp = client.get("/dashboard/traceability-matrix")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["controls"] == []
|
|
assert data["summary"]["total_controls"] == 0
|
|
assert data["summary"]["covered_controls"] == 0
|
|
assert data["summary"]["fully_verified"] == 0
|
|
assert data["summary"]["uncovered_controls"] == 0
|
|
|
|
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
|
@patch("compliance.api.dashboard_routes.ControlRepository")
|
|
def test_nested_structure(self, mock_ctrl_cls, mock_ev_cls):
|
|
"""Control with evidence and assertions should return nested structure."""
|
|
ctrl = make_control(id="c1", control_id="PRIV-001", title="Privacy Control")
|
|
ev = make_evidence(id="e1", control_id="c1", confidence="E3")
|
|
assertion = make_assertion(id="a1", entity_id="e1", verified_by="auditor@example.com")
|
|
|
|
mock_ctrl = MagicMock()
|
|
mock_ctrl.get_all.return_value = [ctrl]
|
|
mock_ctrl_cls.return_value = mock_ctrl
|
|
|
|
mock_ev = MagicMock()
|
|
mock_ev.get_all.return_value = [ev]
|
|
mock_ev_cls.return_value = mock_ev
|
|
|
|
mock_db.reset_mock()
|
|
mock_query = MagicMock()
|
|
mock_query.filter.return_value.all.return_value = [assertion]
|
|
mock_db.query.return_value = mock_query
|
|
|
|
resp = client.get("/dashboard/traceability-matrix")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
|
|
assert len(data["controls"]) == 1
|
|
c = data["controls"][0]
|
|
assert c["control_id"] == "PRIV-001"
|
|
assert len(c["evidence"]) == 1
|
|
assert c["evidence"][0]["confidence_level"] == "E3"
|
|
assert len(c["evidence"][0]["assertions"]) == 1
|
|
assert c["evidence"][0]["assertions"][0]["verified"] is True
|
|
|
|
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
|
@patch("compliance.api.dashboard_routes.ControlRepository")
|
|
def test_assertions_grouped_under_correct_evidence(self, mock_ctrl_cls, mock_ev_cls):
|
|
"""Assertions should only appear under the evidence they reference."""
|
|
ctrl = make_control(id="c1")
|
|
ev1 = make_evidence(id="e1", control_id="c1", title="Evidence A")
|
|
ev2 = make_evidence(id="e2", control_id="c1", title="Evidence B")
|
|
a1 = make_assertion(id="a1", entity_id="e1", sentence_text="Assertion for E1")
|
|
a2 = make_assertion(id="a2", entity_id="e2", sentence_text="Assertion for E2")
|
|
a3 = make_assertion(id="a3", entity_id="e2", sentence_text="Second assertion for E2")
|
|
|
|
mock_ctrl = MagicMock()
|
|
mock_ctrl.get_all.return_value = [ctrl]
|
|
mock_ctrl_cls.return_value = mock_ctrl
|
|
|
|
mock_ev = MagicMock()
|
|
mock_ev.get_all.return_value = [ev1, ev2]
|
|
mock_ev_cls.return_value = mock_ev
|
|
|
|
mock_db.reset_mock()
|
|
mock_query = MagicMock()
|
|
mock_query.filter.return_value.all.return_value = [a1, a2, a3]
|
|
mock_db.query.return_value = mock_query
|
|
|
|
resp = client.get("/dashboard/traceability-matrix")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
|
|
c = data["controls"][0]
|
|
ev1_data = next(e for e in c["evidence"] if e["id"] == "e1")
|
|
ev2_data = next(e for e in c["evidence"] if e["id"] == "e2")
|
|
assert len(ev1_data["assertions"]) == 1
|
|
assert len(ev2_data["assertions"]) == 2
|
|
|
|
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
|
@patch("compliance.api.dashboard_routes.ControlRepository")
|
|
def test_coverage_flags_correct(self, mock_ctrl_cls, mock_ev_cls):
|
|
"""Coverage flags should reflect evidence, assertions, and verification state."""
|
|
ctrl = make_control(id="c1")
|
|
ev = make_evidence(id="e1", control_id="c1", confidence="E2")
|
|
# One verified, one not
|
|
a1 = make_assertion(id="a1", entity_id="e1", verified_by="alice")
|
|
a2 = make_assertion(id="a2", entity_id="e1", verified_by=None)
|
|
|
|
mock_ctrl = MagicMock()
|
|
mock_ctrl.get_all.return_value = [ctrl]
|
|
mock_ctrl_cls.return_value = mock_ctrl
|
|
|
|
mock_ev = MagicMock()
|
|
mock_ev.get_all.return_value = [ev]
|
|
mock_ev_cls.return_value = mock_ev
|
|
|
|
mock_db.reset_mock()
|
|
mock_query = MagicMock()
|
|
mock_query.filter.return_value.all.return_value = [a1, a2]
|
|
mock_db.query.return_value = mock_query
|
|
|
|
resp = client.get("/dashboard/traceability-matrix")
|
|
assert resp.status_code == 200
|
|
|
|
cov = resp.json()["controls"][0]["coverage"]
|
|
assert cov["has_evidence"] is True
|
|
assert cov["has_assertions"] is True
|
|
assert cov["all_assertions_verified"] is False # a2 not verified
|
|
assert cov["min_confidence_level"] == "E2"
|
|
|
|
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
|
@patch("compliance.api.dashboard_routes.ControlRepository")
|
|
def test_coverage_without_evidence(self, mock_ctrl_cls, mock_ev_cls):
|
|
"""Control with no evidence should have all coverage flags False/None."""
|
|
ctrl = make_control(id="c1")
|
|
|
|
mock_ctrl = MagicMock()
|
|
mock_ctrl.get_all.return_value = [ctrl]
|
|
mock_ctrl_cls.return_value = mock_ctrl
|
|
|
|
mock_ev = MagicMock()
|
|
mock_ev.get_all.return_value = []
|
|
mock_ev_cls.return_value = mock_ev
|
|
|
|
mock_db.reset_mock()
|
|
mock_query = MagicMock()
|
|
mock_query.filter.return_value.all.return_value = []
|
|
mock_db.query.return_value = mock_query
|
|
|
|
resp = client.get("/dashboard/traceability-matrix")
|
|
assert resp.status_code == 200
|
|
|
|
cov = resp.json()["controls"][0]["coverage"]
|
|
assert cov["has_evidence"] is False
|
|
assert cov["has_assertions"] is False
|
|
assert cov["all_assertions_verified"] is False
|
|
assert cov["min_confidence_level"] is None
|
|
|
|
@patch("compliance.api.dashboard_routes.EvidenceRepository")
|
|
@patch("compliance.api.dashboard_routes.ControlRepository")
|
|
def test_summary_counts(self, mock_ctrl_cls, mock_ev_cls):
|
|
"""Summary should count total, covered, fully verified, and uncovered controls."""
|
|
# c1: has evidence + verified assertions → fully verified
|
|
# c2: has evidence but no assertions → covered, not fully verified
|
|
# c3: no evidence → uncovered
|
|
c1 = make_control(id="c1", control_id="C-001")
|
|
c2 = make_control(id="c2", control_id="C-002")
|
|
c3 = make_control(id="c3", control_id="C-003")
|
|
|
|
ev1 = make_evidence(id="e1", control_id="c1", confidence="E3")
|
|
ev2 = make_evidence(id="e2", control_id="c2", confidence="E1")
|
|
|
|
a1 = make_assertion(id="a1", entity_id="e1", verified_by="auditor")
|
|
|
|
mock_ctrl = MagicMock()
|
|
mock_ctrl.get_all.return_value = [c1, c2, c3]
|
|
mock_ctrl_cls.return_value = mock_ctrl
|
|
|
|
mock_ev = MagicMock()
|
|
mock_ev.get_all.return_value = [ev1, ev2]
|
|
mock_ev_cls.return_value = mock_ev
|
|
|
|
mock_db.reset_mock()
|
|
mock_query = MagicMock()
|
|
mock_query.filter.return_value.all.return_value = [a1]
|
|
mock_db.query.return_value = mock_query
|
|
|
|
resp = client.get("/dashboard/traceability-matrix")
|
|
assert resp.status_code == 200
|
|
|
|
summary = resp.json()["summary"]
|
|
assert summary["total_controls"] == 3
|
|
assert summary["covered_controls"] == 2
|
|
assert summary["fully_verified"] == 1
|
|
assert summary["uncovered_controls"] == 1
|