feat(completeness): Regulatory Completeness Engine — auditable coverage, not confidence
Phase A½. The move from feature to product development: for every assessment, answer "how sure are we that this answer is COMPLETE?" — different from confidence. The product never claims full coverage; it makes its own knowledge state transparent and auditable. Shows what we do NOT know and why. - compliance/completeness/: assess_completeness(identified, corpus_status, uncertain, assumptions, assessed_obligations) -> CompletenessReport. Separates IDENTIFIED from ASSESSED (validated corpus AND determined applicability) and justifies every gap. Two kinds of open: corpus gap (future_corpus) and applicability uncertainty (query_required + deciding question, e.g. Data Act / generates_usage_data). - The metric is COUNTS, never a single percentage: "Identifiziert N · bewertet M · offen K · Unsicherheiten U · Begründung ja" + an honest audit statement. - ADR-007: auditable honesty; phase order A factory -> A½ Completeness -> B new domains; the transparency selling point. Deterministic, no LLM; corpus status + obligation count injected. - reference suite: "Regulatory Completeness" section runs an industrial-dishwasher assessment (assessed CRA/MaschinenVO; open EMV/Environmental=future_corpus, Data Act=query_required) and notes Environmental flips open->validated automatically once the corpus lands. 11 completeness tests (54 with adjacent modules), mypy --strict clean (15 files), check-loc 0. Product code with no app caller + ADR/reference = non-runtime -> no deploy (ADR-001). Freeze-safe. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
"""Tests for the Regulatory Completeness Engine — auditable coverage, not confidence.
|
||||
|
||||
Acceptance: separate identified from assessed regulations; justify every gap (corpus gap ->
|
||||
future_corpus, applicability uncertain -> query_required with a deciding question); report counts
|
||||
(never a single percentage); emit an honest audit statement. The product shows what it does NOT
|
||||
know and why.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from compliance.completeness import CompletenessReport, CorpusStatus, assess_completeness
|
||||
|
||||
IDENTIFIED = ["CRA", "MaschinenVO", "EMV", "Environmental", "DataAct"]
|
||||
CORPUS = {"CRA": "validated", "MaschinenVO": "validated", "EMV": "validated",
|
||||
"Environmental": "unsupported", "DataAct": "validated"}
|
||||
UNCERTAIN = [{"regulation": "DataAct", "deciding_question": "generates_usage_data", "reason": "generates_usage_data unbekannt"}]
|
||||
|
||||
|
||||
def _report():
|
||||
return assess_completeness(IDENTIFIED, CORPUS, uncertain=UNCERTAIN,
|
||||
assumptions=[{"key": "funkmodul", "value": "nein"}], assessed_obligations=128)
|
||||
|
||||
|
||||
def test_assessed_excludes_uncertain_even_if_corpus_validated():
|
||||
r = _report()
|
||||
# DataAct has a validated corpus but uncertain applicability -> NOT assessed
|
||||
assert r.assessed_regulations == ["CRA", "EMV", "MaschinenVO"]
|
||||
assert "DataAct" in r.open_regulations and "Environmental" in r.open_regulations
|
||||
|
||||
|
||||
def test_corpus_gap_vs_applicability_exclusion():
|
||||
r = _report()
|
||||
by = {e.subject: e for e in r.exclusions}
|
||||
assert by["DataAct"].resolution == "query_required" and by["DataAct"].deciding_question == "generates_usage_data"
|
||||
assert by["Environmental"].resolution == "future_corpus"
|
||||
|
||||
|
||||
def test_open_corpora_is_unsupported_only():
|
||||
r = _report()
|
||||
# DataAct corpus is validated (only applicability is open) -> NOT an open corpus
|
||||
assert r.open_corpora == ["Environmental"]
|
||||
|
||||
|
||||
def test_justification_present_when_every_gap_has_a_reason():
|
||||
r = _report()
|
||||
assert r.justification_present is True
|
||||
open_subjects = {e.subject for e in r.exclusions}
|
||||
assert set(r.open_regulations) <= open_subjects
|
||||
|
||||
|
||||
def test_counts_summary_has_no_percentage():
|
||||
r = _report()
|
||||
assert "%" not in r.completeness_summary and "%" not in r.audit_statement
|
||||
assert "Identifiziert 5" in r.completeness_summary and "bewertet 3" in r.completeness_summary
|
||||
|
||||
|
||||
def test_audit_statement_is_honest():
|
||||
r = _report()
|
||||
assert "3 von 5" in r.audit_statement and "bewusst nicht bewertet" in r.audit_statement
|
||||
|
||||
|
||||
def test_draft_corpus_is_in_review_exclusion():
|
||||
r = assess_completeness(["CRA", "IEC62443"], {"CRA": "validated", "IEC62443": "draft"})
|
||||
by = {e.subject: e for e in r.exclusions}
|
||||
assert by["IEC62443"].resolution == "in_review"
|
||||
assert "IEC62443" in r.open_regulations and "IEC62443" not in r.open_corpora # draft != missing corpus
|
||||
|
||||
|
||||
def test_all_assessed_no_open():
|
||||
r = assess_completeness(["CRA", "MaschinenVO"], {"CRA": "validated", "MaschinenVO": "validated"})
|
||||
assert r.open_regulations == [] and r.exclusions == []
|
||||
assert r.justification_present is True
|
||||
assert "alle 2" in r.audit_statement
|
||||
|
||||
|
||||
def test_coverage_status_mapped():
|
||||
r = _report()
|
||||
cov = {c.regulation: c.status for c in r.coverage}
|
||||
assert cov["CRA"] == CorpusStatus.VALIDATED and cov["Environmental"] == CorpusStatus.UNSUPPORTED
|
||||
|
||||
|
||||
def test_assumptions_and_obligations_carried():
|
||||
r = _report()
|
||||
assert r.assessed_obligations == 128
|
||||
assert [a.key for a in r.assumptions] == ["funkmodul"]
|
||||
|
||||
|
||||
def test_unknown_regulation_defaults_to_open_corpus():
|
||||
r = assess_completeness(["CRA", "REACH"], {"CRA": "validated"}) # REACH not in registry -> unknown
|
||||
assert "REACH" in r.open_corpora and r.assessed_regulations == ["CRA"]
|
||||
|
||||
|
||||
def test_deterministic_and_type():
|
||||
r1 = _report()
|
||||
r2 = _report()
|
||||
assert r1.model_dump() == r2.model_dump()
|
||||
assert isinstance(r1, CompletenessReport)
|
||||
Reference in New Issue
Block a user