feat: Anti-Fake-Evidence System (Phase 1-4b)
Implement full evidence integrity pipeline to prevent compliance theater: - Confidence levels (E0-E4), truth status tracking, assertion engine - Four-Eyes approval workflow, audit trail, reject endpoint - Evidence distribution dashboard, LLM audit routes - Traceability matrix (backend endpoint + Compliance Hub UI tab) - Anti-fake badges, control status machine, normative patterns - 2 migrations, 4 test suites, MkDocs documentation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
125
backend-compliance/migrations/076_anti_fake_evidence.sql
Normal file
125
backend-compliance/migrations/076_anti_fake_evidence.sql
Normal file
@@ -0,0 +1,125 @@
|
||||
-- Migration 076: Anti-Fake-Evidence Guardrails (Phase 1)
|
||||
--
|
||||
-- Prevents "Compliance-Theater": generated content passed off as real evidence,
|
||||
-- controls without evidence marked as "pass", unvalidated 100% compliance claims.
|
||||
--
|
||||
-- Changes:
|
||||
-- 1. New ENUM types for evidence confidence + truth status
|
||||
-- 2. New columns on compliance_evidence (confidence, truth, review tracking)
|
||||
-- 3. New value 'in_progress' for controlstatusenum
|
||||
-- 4. status_justification column on compliance_controls
|
||||
-- 5. New table compliance_llm_generation_audit
|
||||
-- 6. Backfill existing evidence based on source
|
||||
-- 7. Indexes on new columns
|
||||
|
||||
-- ============================================================================
|
||||
-- 1. New ENUM types
|
||||
-- ============================================================================
|
||||
|
||||
-- NOTE: CREATE TYPE cannot run inside a transaction block when combined with
|
||||
-- ALTER TYPE ... ADD VALUE. Each statement here is auto-committed separately
|
||||
-- when executed outside a transaction (which is the default for psql scripts).
|
||||
|
||||
CREATE TYPE evidence_confidence_level AS ENUM (
|
||||
'E0', -- Generated / no real evidence (LLM output, placeholder)
|
||||
'E1', -- Uploaded but unreviewed (manual upload, no hash, no reviewer)
|
||||
'E2', -- Reviewed internally (human reviewed, hash verified)
|
||||
'E3', -- Observed by system (CI/CD pipeline, API with hash)
|
||||
'E4' -- Validated by external auditor
|
||||
);
|
||||
|
||||
CREATE TYPE evidence_truth_status AS ENUM (
|
||||
'generated', -- Created by LLM / system generation
|
||||
'uploaded', -- Manually uploaded by user
|
||||
'observed', -- Automatically observed (CI/CD, monitoring)
|
||||
'validated_internal', -- Reviewed + approved by internal reviewer
|
||||
'rejected', -- Reviewed and rejected
|
||||
'provided_to_auditor', -- Shared with external auditor
|
||||
'accepted_by_auditor' -- Accepted by external auditor
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 2. Add 'in_progress' to controlstatusenum
|
||||
-- ============================================================================
|
||||
-- ALTER TYPE ... ADD VALUE cannot run inside a transaction.
|
||||
|
||||
ALTER TYPE controlstatusenum ADD VALUE IF NOT EXISTS 'in_progress';
|
||||
|
||||
-- ============================================================================
|
||||
-- 3. New columns on compliance_evidence
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE compliance_evidence
|
||||
ADD COLUMN IF NOT EXISTS confidence_level evidence_confidence_level DEFAULT 'E1',
|
||||
ADD COLUMN IF NOT EXISTS truth_status evidence_truth_status DEFAULT 'uploaded',
|
||||
ADD COLUMN IF NOT EXISTS generation_mode VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS may_be_used_as_evidence BOOLEAN DEFAULT TRUE,
|
||||
ADD COLUMN IF NOT EXISTS reviewed_by VARCHAR(200),
|
||||
ADD COLUMN IF NOT EXISTS reviewed_at TIMESTAMPTZ;
|
||||
|
||||
-- ============================================================================
|
||||
-- 4. status_justification on compliance_controls
|
||||
-- ============================================================================
|
||||
|
||||
ALTER TABLE compliance_controls
|
||||
ADD COLUMN IF NOT EXISTS status_justification TEXT;
|
||||
|
||||
-- ============================================================================
|
||||
-- 5. LLM Generation Audit table
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS compliance_llm_generation_audit (
|
||||
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
||||
tenant_id VARCHAR(36),
|
||||
entity_type VARCHAR(50) NOT NULL, -- 'evidence', 'control', 'document', ...
|
||||
entity_id VARCHAR(36), -- FK to the generated entity
|
||||
generation_mode VARCHAR(100) NOT NULL, -- 'draft_assistance', 'auto_generation', ...
|
||||
truth_status evidence_truth_status NOT NULL DEFAULT 'generated',
|
||||
may_be_used_as_evidence BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
llm_model VARCHAR(100),
|
||||
llm_provider VARCHAR(50), -- 'ollama', 'anthropic', ...
|
||||
prompt_hash VARCHAR(64), -- SHA-256 of the prompt
|
||||
input_summary TEXT, -- Truncated input for auditability
|
||||
output_summary TEXT, -- Truncated output for auditability
|
||||
metadata JSONB DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- ============================================================================
|
||||
-- 6. Backfill existing evidence based on source
|
||||
-- ============================================================================
|
||||
|
||||
-- CI pipeline evidence → E3 + observed
|
||||
UPDATE compliance_evidence
|
||||
SET confidence_level = 'E3',
|
||||
truth_status = 'observed'
|
||||
WHERE source = 'ci_pipeline'
|
||||
AND confidence_level = 'E1';
|
||||
|
||||
-- API evidence → E3 + observed
|
||||
UPDATE compliance_evidence
|
||||
SET confidence_level = 'E3',
|
||||
truth_status = 'observed'
|
||||
WHERE source = 'api'
|
||||
AND confidence_level = 'E1';
|
||||
|
||||
-- Manual/upload evidence stays at E1 + uploaded (default)
|
||||
|
||||
-- Generated evidence → E0 + generated
|
||||
UPDATE compliance_evidence
|
||||
SET confidence_level = 'E0',
|
||||
truth_status = 'generated',
|
||||
may_be_used_as_evidence = FALSE
|
||||
WHERE source = 'generated'
|
||||
AND confidence_level = 'E1';
|
||||
|
||||
-- ============================================================================
|
||||
-- 7. Indexes
|
||||
-- ============================================================================
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_confidence ON compliance_evidence (confidence_level);
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_truth_status ON compliance_evidence (truth_status);
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_may_be_used ON compliance_evidence (may_be_used_as_evidence);
|
||||
CREATE INDEX IF NOT EXISTS ix_llm_audit_entity ON compliance_llm_generation_audit (entity_type, entity_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_llm_audit_tenant ON compliance_llm_generation_audit (tenant_id);
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Migration 077: Anti-Fake-Evidence Phase 2
|
||||
-- Assertions table, Four-Eyes columns on Evidence, Audit-Trail performance index
|
||||
|
||||
-- 1A. Assertions table
|
||||
CREATE TABLE IF NOT EXISTS compliance_assertions (
|
||||
id VARCHAR(36) PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
||||
tenant_id VARCHAR(36),
|
||||
entity_type VARCHAR(50) NOT NULL,
|
||||
entity_id VARCHAR(36) NOT NULL,
|
||||
sentence_text TEXT NOT NULL,
|
||||
sentence_index INTEGER NOT NULL DEFAULT 0,
|
||||
assertion_type VARCHAR(20) NOT NULL DEFAULT 'assertion',
|
||||
evidence_ids JSONB DEFAULT '[]'::jsonb,
|
||||
confidence FLOAT DEFAULT 0.0,
|
||||
normative_tier VARCHAR(20),
|
||||
verified_by VARCHAR(200),
|
||||
verified_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_assertion_entity ON compliance_assertions (entity_type, entity_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_assertion_type ON compliance_assertions (assertion_type);
|
||||
CREATE INDEX IF NOT EXISTS ix_assertion_tenant ON compliance_assertions (tenant_id);
|
||||
|
||||
-- 1B. Four-Eyes columns on Evidence
|
||||
ALTER TABLE compliance_evidence
|
||||
ADD COLUMN IF NOT EXISTS approval_status VARCHAR(30) DEFAULT 'none',
|
||||
ADD COLUMN IF NOT EXISTS first_reviewer VARCHAR(200),
|
||||
ADD COLUMN IF NOT EXISTS first_reviewed_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS second_reviewer VARCHAR(200),
|
||||
ADD COLUMN IF NOT EXISTS second_reviewed_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS requires_four_eyes BOOLEAN DEFAULT FALSE;
|
||||
CREATE INDEX IF NOT EXISTS ix_evidence_approval_status ON compliance_evidence (approval_status);
|
||||
|
||||
-- 1C. Audit-Trail performance index
|
||||
CREATE INDEX IF NOT EXISTS ix_audit_trail_entity_action
|
||||
ON compliance_audit_trail (entity_type, action, performed_at);
|
||||
Reference in New Issue
Block a user