Files
breakpilot-compliance/ai-compliance-sdk/internal/iace/completeness_test.go
Benjamin Boenisch 06711bad1c
All checks were successful
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-ai-compliance (push) Successful in 44s
CI / test-python-backend-compliance (push) Successful in 37s
CI / test-python-document-crawler (push) Successful in 22s
CI / test-python-dsms-gateway (push) Successful in 20s
feat(sdk,iace): add Personalized Drafting Pipeline v2 and IACE engine
Drafting Engine: 7-module pipeline with narrative tags, allowed facts governance,
PII sanitizer, prose validator with repair loop, hash-based cache, and terminology
guide. v1 fallback via ?v=1 query param.

IACE: Initial AI-Act Conformity Engine with risk classifier, completeness checker,
hazard library, and PostgreSQL store for AI system assessments.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-25 22:27:06 +01:00

679 lines
18 KiB
Go

package iace
import (
"encoding/json"
"math"
"testing"
"github.com/google/uuid"
)
// helper to build metadata with the given keys set to non-empty string values.
func metadataWith(keys ...string) json.RawMessage {
m := make(map[string]interface{})
for _, k := range keys {
m[k] = "defined"
}
data, _ := json.Marshal(m)
return data
}
func TestCompletenessCheck_EmptyContext(t *testing.T) {
checker := NewCompletenessChecker()
ctx := &CompletenessContext{
Project: nil,
}
result := checker.Check(ctx)
if result.CanExport {
t.Error("CanExport should be false for empty context")
}
// With nil project, most gates fail. However, some auto-pass:
// G06 (AI classification): auto-passes when HasAI=false
// G22 (critical/high mitigated): auto-passes when no critical/high assessments exist
// G23 (mitigations verified): auto-passes when no mitigations with status "implemented"
// G42 (AI documents): auto-passes when HasAI=false
// That gives 4 required gates passing even with empty context.
if result.PassedRequired != 4 {
t.Errorf("PassedRequired = %d, want 4 (G06, G22, G23, G42 auto-pass)", result.PassedRequired)
}
// Score should be low: 4/20 * 80 = 16
if result.Score > 20 {
t.Errorf("Score = %f, expected <= 20 for empty context", result.Score)
}
if len(result.Gates) == 0 {
t.Error("Gates should not be empty")
}
}
func TestCompletenessCheck_MinimalValidProject(t *testing.T) {
checker := NewCompletenessChecker()
projectID := uuid.New()
hazardID := uuid.New()
componentID := uuid.New()
ctx := &CompletenessContext{
Project: &Project{
ID: projectID,
MachineName: "TestMachine",
Description: "A test machine for unit testing",
Manufacturer: "TestCorp",
CEMarkingTarget: "2023/1230",
Metadata: metadataWith("operating_limits", "foreseeable_misuse"),
},
Components: []Component{
{ID: componentID, Name: "SafetyPLC", ComponentType: ComponentTypeSoftware, IsSafetyRelevant: true},
},
Classifications: []RegulatoryClassification{
{Regulation: RegulationAIAct},
{Regulation: RegulationMachineryRegulation},
{Regulation: RegulationNIS2},
{Regulation: RegulationCRA},
},
Hazards: []Hazard{
{ID: hazardID, ProjectID: projectID, ComponentID: componentID, Name: "TestHazard", Category: "test"},
},
Assessments: []RiskAssessment{
{ID: uuid.New(), HazardID: hazardID, RiskLevel: RiskLevelLow, IsAcceptable: true},
},
Mitigations: []Mitigation{
{ID: uuid.New(), HazardID: hazardID, Status: MitigationStatusVerified},
},
Evidence: []Evidence{
{ID: uuid.New(), ProjectID: projectID, FileName: "test.pdf"},
},
TechFileSections: []TechFileSection{
{ID: uuid.New(), ProjectID: projectID, SectionType: "risk_assessment_report"},
{ID: uuid.New(), ProjectID: projectID, SectionType: "hazard_log_combined"},
},
HasAI: false,
}
result := checker.Check(ctx)
if !result.CanExport {
t.Error("CanExport should be true for fully valid project")
for _, g := range result.Gates {
if g.Required && !g.Passed {
t.Errorf(" Required gate %s (%s) not passed: %s", g.ID, g.Label, g.Details)
}
}
}
if result.PassedRequired != result.TotalRequired {
t.Errorf("PassedRequired = %d, TotalRequired = %d, want all passed", result.PassedRequired, result.TotalRequired)
}
// Score should be at least 80 (all required) + 15 (evidence recommended) = 95
if result.Score < 80 {
t.Errorf("Score = %f, expected >= 80 for fully valid project", result.Score)
}
}
func TestCompletenessCheck_PartialRequiredGates(t *testing.T) {
checker := NewCompletenessChecker()
// Provide only some required data: machine name, manufacturer, description, one component, one safety-relevant.
// Missing: operating_limits, foreseeable_misuse, classifications, hazards, assessments, tech files.
ctx := &CompletenessContext{
Project: &Project{
MachineName: "PartialMachine",
Description: "Some description",
Manufacturer: "TestCorp",
},
Components: []Component{
{Name: "Sensor", ComponentType: ComponentTypeSensor, IsSafetyRelevant: true},
},
HasAI: false,
}
result := checker.Check(ctx)
if result.CanExport {
t.Error("CanExport should be false when not all required gates pass")
}
if result.PassedRequired == 0 {
t.Error("Some required gates should pass (G01, G02, G05, G06, G07, G08)")
}
if result.PassedRequired >= result.TotalRequired {
t.Errorf("PassedRequired (%d) should be less than TotalRequired (%d)", result.PassedRequired, result.TotalRequired)
}
// Score should be partial
if result.Score <= 0 || result.Score >= 95 {
t.Errorf("Score = %f, expected partial score between 0 and 95", result.Score)
}
}
func TestCompletenessCheck_G06_AIClassificationGate(t *testing.T) {
checker := NewCompletenessChecker()
tests := []struct {
name string
hasAI bool
classifications []RegulatoryClassification
wantG06Passed bool
}{
{
name: "no AI present auto-passes G06",
hasAI: false,
classifications: nil,
wantG06Passed: true,
},
{
name: "AI present without classification fails G06",
hasAI: true,
classifications: nil,
wantG06Passed: false,
},
{
name: "AI present with AI Act classification passes G06",
hasAI: true,
classifications: []RegulatoryClassification{
{Regulation: RegulationAIAct},
},
wantG06Passed: true,
},
{
name: "AI present with non-AI classification fails G06",
hasAI: true,
classifications: []RegulatoryClassification{
{Regulation: RegulationCRA},
},
wantG06Passed: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := &CompletenessContext{
Project: &Project{MachineName: "Test"},
Classifications: tt.classifications,
HasAI: tt.hasAI,
}
result := checker.Check(ctx)
for _, g := range result.Gates {
if g.ID == "G06" {
if g.Passed != tt.wantG06Passed {
t.Errorf("G06 Passed = %v, want %v", g.Passed, tt.wantG06Passed)
}
return
}
}
t.Error("G06 gate not found in results")
})
}
}
func TestCompletenessCheck_G42_AIDocumentsGate(t *testing.T) {
checker := NewCompletenessChecker()
tests := []struct {
name string
hasAI bool
techFileSections []TechFileSection
wantG42Passed bool
}{
{
name: "no AI auto-passes G42",
hasAI: false,
techFileSections: nil,
wantG42Passed: true,
},
{
name: "AI present without tech files fails G42",
hasAI: true,
techFileSections: nil,
wantG42Passed: false,
},
{
name: "AI present with only intended_purpose fails G42",
hasAI: true,
techFileSections: []TechFileSection{
{SectionType: "ai_intended_purpose"},
},
wantG42Passed: false,
},
{
name: "AI present with only model_description fails G42",
hasAI: true,
techFileSections: []TechFileSection{
{SectionType: "ai_model_description"},
},
wantG42Passed: false,
},
{
name: "AI present with both AI sections passes G42",
hasAI: true,
techFileSections: []TechFileSection{
{SectionType: "ai_intended_purpose"},
{SectionType: "ai_model_description"},
},
wantG42Passed: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := &CompletenessContext{
Project: &Project{MachineName: "Test"},
TechFileSections: tt.techFileSections,
HasAI: tt.hasAI,
}
result := checker.Check(ctx)
for _, g := range result.Gates {
if g.ID == "G42" {
if g.Passed != tt.wantG42Passed {
t.Errorf("G42 Passed = %v, want %v", g.Passed, tt.wantG42Passed)
}
return
}
}
t.Error("G42 gate not found in results")
})
}
}
func TestCompletenessCheck_G22_CriticalHighMitigated(t *testing.T) {
checker := NewCompletenessChecker()
hazardID := uuid.New()
tests := []struct {
name string
assessments []RiskAssessment
mitigations []Mitigation
wantG22Passed bool
}{
{
name: "no critical/high hazards auto-passes G22",
assessments: []RiskAssessment{{HazardID: hazardID, RiskLevel: RiskLevelLow}},
mitigations: nil,
wantG22Passed: true,
},
{
name: "no assessments at all auto-passes G22 (no critical/high found)",
assessments: nil,
mitigations: nil,
wantG22Passed: true,
},
{
name: "critical hazard without mitigation fails G22",
assessments: []RiskAssessment{{HazardID: hazardID, RiskLevel: RiskLevelCritical}},
mitigations: nil,
wantG22Passed: false,
},
{
name: "high hazard without mitigation fails G22",
assessments: []RiskAssessment{{HazardID: hazardID, RiskLevel: RiskLevelHigh}},
mitigations: nil,
wantG22Passed: false,
},
{
name: "critical hazard with mitigation passes G22",
assessments: []RiskAssessment{{HazardID: hazardID, RiskLevel: RiskLevelCritical}},
mitigations: []Mitigation{{HazardID: hazardID, Status: MitigationStatusVerified}},
wantG22Passed: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := &CompletenessContext{
Project: &Project{MachineName: "Test"},
Assessments: tt.assessments,
Mitigations: tt.mitigations,
}
result := checker.Check(ctx)
for _, g := range result.Gates {
if g.ID == "G22" {
if g.Passed != tt.wantG22Passed {
t.Errorf("G22 Passed = %v, want %v", g.Passed, tt.wantG22Passed)
}
return
}
}
t.Error("G22 gate not found in results")
})
}
}
func TestCompletenessCheck_G23_MitigationsVerified(t *testing.T) {
checker := NewCompletenessChecker()
hazardID := uuid.New()
tests := []struct {
name string
mitigations []Mitigation
wantG23Passed bool
}{
{
name: "no mitigations passes G23",
mitigations: nil,
wantG23Passed: true,
},
{
name: "all mitigations verified passes G23",
mitigations: []Mitigation{
{HazardID: hazardID, Status: MitigationStatusVerified},
{HazardID: hazardID, Status: MitigationStatusVerified},
},
wantG23Passed: true,
},
{
name: "one mitigation still implemented fails G23",
mitigations: []Mitigation{
{HazardID: hazardID, Status: MitigationStatusVerified},
{HazardID: hazardID, Status: MitigationStatusImplemented},
},
wantG23Passed: false,
},
{
name: "planned mitigations pass G23 (not yet implemented)",
mitigations: []Mitigation{
{HazardID: hazardID, Status: MitigationStatusPlanned},
},
wantG23Passed: true,
},
{
name: "rejected mitigations pass G23",
mitigations: []Mitigation{
{HazardID: hazardID, Status: MitigationStatusRejected},
},
wantG23Passed: true,
},
{
name: "mix of verified planned rejected passes G23",
mitigations: []Mitigation{
{HazardID: hazardID, Status: MitigationStatusVerified},
{HazardID: hazardID, Status: MitigationStatusPlanned},
{HazardID: hazardID, Status: MitigationStatusRejected},
},
wantG23Passed: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := &CompletenessContext{
Project: &Project{MachineName: "Test"},
Mitigations: tt.mitigations,
}
result := checker.Check(ctx)
for _, g := range result.Gates {
if g.ID == "G23" {
if g.Passed != tt.wantG23Passed {
t.Errorf("G23 Passed = %v, want %v", g.Passed, tt.wantG23Passed)
}
return
}
}
t.Error("G23 gate not found in results")
})
}
}
func TestCompletenessCheck_G24_ResidualRiskAccepted(t *testing.T) {
checker := NewCompletenessChecker()
hazardID := uuid.New()
tests := []struct {
name string
assessments []RiskAssessment
wantG24Passed bool
}{
{
name: "no assessments fails G24",
assessments: nil,
wantG24Passed: false,
},
{
name: "all assessments acceptable passes G24",
assessments: []RiskAssessment{
{HazardID: hazardID, IsAcceptable: true, RiskLevel: RiskLevelMedium},
{HazardID: hazardID, IsAcceptable: true, RiskLevel: RiskLevelHigh},
},
wantG24Passed: true,
},
{
name: "not acceptable but low risk passes G24",
assessments: []RiskAssessment{
{HazardID: hazardID, IsAcceptable: false, RiskLevel: RiskLevelLow},
},
wantG24Passed: true,
},
{
name: "not acceptable but negligible risk passes G24",
assessments: []RiskAssessment{
{HazardID: hazardID, IsAcceptable: false, RiskLevel: RiskLevelNegligible},
},
wantG24Passed: true,
},
{
name: "not acceptable with high risk fails G24",
assessments: []RiskAssessment{
{HazardID: hazardID, IsAcceptable: false, RiskLevel: RiskLevelHigh},
},
wantG24Passed: false,
},
{
name: "not acceptable with critical risk fails G24",
assessments: []RiskAssessment{
{HazardID: hazardID, IsAcceptable: false, RiskLevel: RiskLevelCritical},
},
wantG24Passed: false,
},
{
name: "not acceptable with medium risk fails G24",
assessments: []RiskAssessment{
{HazardID: hazardID, IsAcceptable: false, RiskLevel: RiskLevelMedium},
},
wantG24Passed: false,
},
{
name: "mix acceptable and unacceptable high fails G24",
assessments: []RiskAssessment{
{HazardID: hazardID, IsAcceptable: true, RiskLevel: RiskLevelHigh},
{HazardID: hazardID, IsAcceptable: false, RiskLevel: RiskLevelHigh},
},
wantG24Passed: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := &CompletenessContext{
Project: &Project{MachineName: "Test"},
Assessments: tt.assessments,
}
result := checker.Check(ctx)
for _, g := range result.Gates {
if g.ID == "G24" {
if g.Passed != tt.wantG24Passed {
t.Errorf("G24 Passed = %v, want %v", g.Passed, tt.wantG24Passed)
}
return
}
}
t.Error("G24 gate not found in results")
})
}
}
func TestCompletenessCheck_ScoringFormula(t *testing.T) {
tests := []struct {
name string
passedRequired int
totalRequired int
passedRecommended int
totalRecommended int
passedOptional int
totalOptional int
wantScore float64
}{
{
name: "all zeros produces zero score",
passedRequired: 0,
totalRequired: 0,
passedRecommended: 0,
totalRecommended: 0,
passedOptional: 0,
totalOptional: 0,
wantScore: 0,
},
{
name: "all required passed gives 80",
passedRequired: 20,
totalRequired: 20,
passedRecommended: 0,
totalRecommended: 1,
passedOptional: 0,
totalOptional: 0,
wantScore: 80,
},
{
name: "half required passed gives 40",
passedRequired: 10,
totalRequired: 20,
passedRecommended: 0,
totalRecommended: 1,
passedOptional: 0,
totalOptional: 0,
wantScore: 40,
},
{
name: "all required and all recommended gives 95",
passedRequired: 20,
totalRequired: 20,
passedRecommended: 1,
totalRecommended: 1,
passedOptional: 0,
totalOptional: 0,
wantScore: 95,
},
{
name: "all categories full gives 100",
passedRequired: 20,
totalRequired: 20,
passedRecommended: 1,
totalRecommended: 1,
passedOptional: 1,
totalOptional: 1,
wantScore: 100,
},
{
name: "only recommended passed",
passedRequired: 0,
totalRequired: 20,
passedRecommended: 1,
totalRecommended: 1,
passedOptional: 0,
totalOptional: 0,
wantScore: 15,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
score := calculateWeightedScore(
tt.passedRequired, tt.totalRequired,
tt.passedRecommended, tt.totalRecommended,
tt.passedOptional, tt.totalOptional,
)
if math.Abs(score-tt.wantScore) > 0.01 {
t.Errorf("calculateWeightedScore = %f, want %f", score, tt.wantScore)
}
})
}
}
func TestCompletenessCheck_GateCountsAndCategories(t *testing.T) {
checker := NewCompletenessChecker()
ctx := &CompletenessContext{
Project: &Project{MachineName: "Test"},
}
result := checker.Check(ctx)
// The buildGateDefinitions function returns exactly 21 gates
// (G01-G08: 8, G10-G13: 4, G20-G24: 5, G30: 1, G40-G42: 3 = 21 total)
if len(result.Gates) != 21 {
t.Errorf("Total gates = %d, want 21", len(result.Gates))
}
// Count required vs recommended
requiredCount := 0
recommendedCount := 0
for _, g := range result.Gates {
if g.Required {
requiredCount++
}
}
// G30 is the only recommended gate (Required=false, Recommended=true)
// All others are required (20 required, 1 recommended)
if requiredCount != 20 {
t.Errorf("Required gates count = %d, want 20", requiredCount)
}
if result.TotalRequired != 20 {
t.Errorf("TotalRequired = %d, want 20", result.TotalRequired)
}
// TotalRecommended should be 1 (G30)
if result.TotalRecommended != 1 {
t.Errorf("TotalRecommended = %d, want 1", result.TotalRecommended)
}
_ = recommendedCount
// Verify expected categories exist
categories := make(map[string]int)
for _, g := range result.Gates {
categories[g.Category]++
}
expectedCategories := map[string]int{
"onboarding": 8,
"classification": 4,
"hazard_risk": 5,
"evidence": 1,
"tech_file": 3,
}
for cat, expectedCount := range expectedCategories {
if categories[cat] != expectedCount {
t.Errorf("Category %q count = %d, want %d", cat, categories[cat], expectedCount)
}
}
}
func TestCompletenessCheck_FailedGateHasDetails(t *testing.T) {
checker := NewCompletenessChecker()
ctx := &CompletenessContext{
Project: &Project{}, // empty project, many gates will fail
}
result := checker.Check(ctx)
for _, g := range result.Gates {
if !g.Passed && g.Details == "" {
t.Errorf("Gate %s failed but has empty Details", g.ID)
}
if g.Passed && g.Details != "" {
t.Errorf("Gate %s passed but has non-empty Details: %s", g.ID, g.Details)
}
}
}