feat(completeness): Regulatory Completeness Engine — auditable coverage, not confidence
Phase A½. The move from feature to product development: for every assessment, answer "how sure are we that this answer is COMPLETE?" — different from confidence. The product never claims full coverage; it makes its own knowledge state transparent and auditable. Shows what we do NOT know and why. - compliance/completeness/: assess_completeness(identified, corpus_status, uncertain, assumptions, assessed_obligations) -> CompletenessReport. Separates IDENTIFIED from ASSESSED (validated corpus AND determined applicability) and justifies every gap. Two kinds of open: corpus gap (future_corpus) and applicability uncertainty (query_required + deciding question, e.g. Data Act / generates_usage_data). - The metric is COUNTS, never a single percentage: "Identifiziert N · bewertet M · offen K · Unsicherheiten U · Begründung ja" + an honest audit statement. - ADR-007: auditable honesty; phase order A factory -> A½ Completeness -> B new domains; the transparency selling point. Deterministic, no LLM; corpus status + obligation count injected. - reference suite: "Regulatory Completeness" section runs an industrial-dishwasher assessment (assessed CRA/MaschinenVO; open EMV/Environmental=future_corpus, Data Act=query_required) and notes Environmental flips open->validated automatically once the corpus lands. 11 completeness tests (54 with adjacent modules), mypy --strict clean (15 files), check-loc 0. Product code with no app caller + ADR/reference = non-runtime -> no deploy (ADR-001). Freeze-safe. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
@@ -0,0 +1,24 @@
|
||||
"""Regulatory Completeness — auditable knowledge coverage, not confidence.
|
||||
|
||||
An internal quality machine: for an assessment it reports identified vs assessed regulations and
|
||||
justifies every open or excluded domain (corpus gap -> future_corpus; applicability uncertain ->
|
||||
query_required). The metric is counts, never a single percentage. The product never claims full
|
||||
coverage — it makes its own knowledge state transparent and auditable. Deterministic, no LLM, no
|
||||
new corpus/meta-model class (freeze v1.0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .engine import assess_completeness
|
||||
from .schemas import (
|
||||
Assumption, CompletenessReport, CorpusStatus, DomainCoverage, Exclusion,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"assess_completeness",
|
||||
"CompletenessReport",
|
||||
"CorpusStatus",
|
||||
"DomainCoverage",
|
||||
"Exclusion",
|
||||
"Assumption",
|
||||
]
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Regulatory Completeness Engine — measure auditable knowledge coverage for an assessment.
|
||||
|
||||
Separates what we IDENTIFIED (triggered regulations) from what we ASSESSED (validated corpus AND
|
||||
determined applicability), and justifies every gap. Two kinds of „open":
|
||||
- corpus gap — no validated corpus yet (e.g. Environmental) -> future_corpus
|
||||
- applicability open — corpus exists but applicability is uncertain (Data Act) -> query_required
|
||||
The metric is COUNTS, never a single percentage. The audit statement says plainly „wir bewerteten M
|
||||
von N Domänen; K sind nicht im validierten Korpus und wurden bewusst nicht bewertet".
|
||||
|
||||
Deterministic, computed-not-stored, no LLM, no new corpus/meta-model class (freeze v1.0). Python 3.9.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .schemas import (
|
||||
Assumption, CompletenessReport, CorpusStatus, DomainCoverage, Exclusion,
|
||||
)
|
||||
|
||||
_VALID = {s.value for s in CorpusStatus}
|
||||
|
||||
|
||||
def _status(corpus_status: Dict[str, str], reg: str) -> CorpusStatus:
|
||||
raw = corpus_status.get(reg, "unknown")
|
||||
return CorpusStatus(raw) if raw in _VALID else CorpusStatus.UNKNOWN
|
||||
|
||||
|
||||
def assess_completeness(
|
||||
identified_regulations: List[str],
|
||||
corpus_status: Dict[str, str],
|
||||
uncertain: Optional[List[Dict[str, Any]]] = None,
|
||||
assumptions: Optional[List[Dict[str, Any]]] = None,
|
||||
assessed_obligations: int = 0,
|
||||
) -> CompletenessReport:
|
||||
"""Build the auditable coverage report.
|
||||
|
||||
`identified_regulations`: triggered/identified for this product. `corpus_status`: regulation ->
|
||||
one of validated/draft/unsupported/unknown (curated/injected corpus registry). `uncertain`:
|
||||
applicability-uncertain regulations [{regulation, deciding_question, reason}]. `assumptions`:
|
||||
[{key, value, note}]. `assessed_obligations`: count from Execution (injected, default 0).
|
||||
"""
|
||||
ids = sorted(set(identified_regulations))
|
||||
unc = uncertain or []
|
||||
unc_subjects = {str(u.get("regulation") or u.get("subject")) for u in unc if (u.get("regulation") or u.get("subject"))}
|
||||
|
||||
coverage = [DomainCoverage(regulation=r, status=_status(corpus_status, r)) for r in ids]
|
||||
assessed = [r for r in ids if _status(corpus_status, r) == CorpusStatus.VALIDATED and r not in unc_subjects]
|
||||
open_regs = [r for r in ids if r not in assessed]
|
||||
open_corpora = [r for r in ids if _status(corpus_status, r) in (CorpusStatus.UNSUPPORTED, CorpusStatus.UNKNOWN)]
|
||||
|
||||
exclusions: List[Exclusion] = []
|
||||
for u in unc:
|
||||
subj = str(u.get("regulation") or u.get("subject") or "")
|
||||
if not subj:
|
||||
continue
|
||||
exclusions.append(Exclusion(
|
||||
subject=subj, reason=str(u.get("reason", "Anwendbarkeit unsicher")),
|
||||
deciding_question=str(u.get("deciding_question", "")), resolution="query_required"))
|
||||
for r in open_regs:
|
||||
if r in unc_subjects:
|
||||
continue
|
||||
st = _status(corpus_status, r)
|
||||
if st == CorpusStatus.DRAFT:
|
||||
exclusions.append(Exclusion(subject=r, reason="Korpus in Bearbeitung (draft)", resolution="in_review"))
|
||||
else:
|
||||
exclusions.append(Exclusion(subject=r, reason="nicht im validierten Korpus", resolution="future_corpus"))
|
||||
|
||||
covered_subjects = {e.subject for e in exclusions}
|
||||
justification = (not open_regs) or set(open_regs) <= covered_subjects
|
||||
assumptions_m = [Assumption(key=str(a.get("key", "")), value=str(a.get("value", "")), note=str(a.get("note", ""))) for a in (assumptions or [])]
|
||||
|
||||
summary = "Identifiziert %d · bewertet %d · offen %d · Unsicherheiten %d · Begründung %s" % (
|
||||
len(ids), len(assessed), len(open_regs), len(unc), "ja" if justification else "nein")
|
||||
if open_regs:
|
||||
audit = (
|
||||
"Für dieses Produkt konnten wir %d von %d identifizierten regulatorischen Domänen vollständig "
|
||||
"bewerten. %d weitere %s noch nicht Bestandteil des validierten Korpus bzw. anwendungsunsicher "
|
||||
"und wurden deshalb bewusst nicht bewertet." % (
|
||||
len(assessed), len(ids), len(open_regs), "ist" if len(open_regs) == 1 else "sind"))
|
||||
else:
|
||||
audit = "Für dieses Produkt konnten wir alle %d identifizierten regulatorischen Domänen vollständig bewerten." % len(ids)
|
||||
|
||||
return CompletenessReport(
|
||||
identified_regulations=ids, assessed_regulations=assessed, open_regulations=open_regs,
|
||||
open_corpora=open_corpora, coverage=coverage, assumptions=assumptions_m, exclusions=exclusions,
|
||||
uncertainties_count=len(unc), assessed_obligations=assessed_obligations,
|
||||
justification_present=justification, completeness_summary=summary, audit_statement=audit,
|
||||
)
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Schemas for the Regulatory Completeness Engine — auditable knowledge-coverage, not confidence.
|
||||
|
||||
For an assessment it answers „wie sicher sind wir, dass diese Antwort VOLLSTÄNDIG ist?" by separating
|
||||
IDENTIFIED regulations from ASSESSED ones (those in the validated corpus) and listing every open or
|
||||
excluded domain WITH a reason. The metric is counts, never a single „87%". This is an internal quality
|
||||
machine: the product never claims full coverage — it makes its own knowledge state transparent.
|
||||
Deterministic, computed-not-stored, no new meta-model class (freeze v1.0). Python 3.9 compatible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class CorpusStatus(str, Enum):
|
||||
"""The maturity of our knowledge corpus for a regulation/domain."""
|
||||
|
||||
VALIDATED = "validated" # we can fully assess this
|
||||
DRAFT = "draft" # partial / under review
|
||||
UNSUPPORTED = "unsupported" # triggered but no corpus yet
|
||||
UNKNOWN = "unknown" # not in our registry at all
|
||||
|
||||
|
||||
class DomainCoverage(BaseModel):
|
||||
regulation: str
|
||||
status: CorpusStatus = CorpusStatus.UNKNOWN
|
||||
note: str = ""
|
||||
|
||||
|
||||
class Exclusion(BaseModel):
|
||||
"""A domain/regulation DELIBERATELY not assessed — always with a reason (the heart of the engine)."""
|
||||
|
||||
subject: str
|
||||
reason: str
|
||||
deciding_question: str = "" # what would resolve it (if a query)
|
||||
resolution: str = "future_corpus" # query_required | future_corpus | not_applicable
|
||||
|
||||
|
||||
class Assumption(BaseModel):
|
||||
key: str
|
||||
value: str = ""
|
||||
note: str = ""
|
||||
|
||||
|
||||
class CompletenessReport(BaseModel):
|
||||
"""The auditable coverage report for one assessment — counts + justification, NO single percentage."""
|
||||
|
||||
identified_regulations: List[str] = Field(default_factory=list)
|
||||
assessed_regulations: List[str] = Field(default_factory=list) # in the validated corpus
|
||||
open_regulations: List[str] = Field(default_factory=list) # identified but not validated
|
||||
open_corpora: List[str] = Field(default_factory=list) # missing domains worth building
|
||||
coverage: List[DomainCoverage] = Field(default_factory=list)
|
||||
assumptions: List[Assumption] = Field(default_factory=list)
|
||||
exclusions: List[Exclusion] = Field(default_factory=list)
|
||||
uncertainties_count: int = 0
|
||||
assessed_obligations: int = 0 # injected (Execution-owned)
|
||||
justification_present: bool = False
|
||||
completeness_summary: str = "" # "Identifiziert N · bewertet M · offen K · ..."
|
||||
audit_statement: str = "" # the honest narrative sentence
|
||||
@@ -104,3 +104,38 @@ def knowledge_intake_section(base_dir) -> None:
|
||||
("Impact-Triage (HIGH/LOW/NONE/new_domain)", "PASS", "3 Beispiel-Dokumente korrekt eingeordnet"),
|
||||
("Regelwerk-ID-Normalisierung", "TODO", "CRA vs Cyber Resilience Act vereinheitlichen"),
|
||||
])
|
||||
|
||||
|
||||
def completeness_section() -> None:
|
||||
"""Render the Regulatory Completeness section (kept here so generate.py stays under the LOC budget)."""
|
||||
from compliance.completeness import assess_completeness
|
||||
|
||||
rep = assess_completeness(
|
||||
identified_regulations=["CRA", "MaschinenVO", "EMV", "Environmental", "DataAct"],
|
||||
corpus_status={"CRA": "validated", "MaschinenVO": "validated", "EMV": "unsupported",
|
||||
"Environmental": "unsupported", "DataAct": "validated"},
|
||||
uncertain=[{"regulation": "DataAct", "deciding_question": "generates_usage_data", "reason": "generates_usage_data = unbekannt"}],
|
||||
assumptions=[{"key": "Funkmodul", "value": "nein"}, {"key": "personenbezogene Nutzungsdaten", "value": "nein"}],
|
||||
assessed_obligations=128)
|
||||
w("## Regulatory Completeness — was wir bewerten konnten, und was bewusst nicht")
|
||||
w("")
|
||||
w('_Interne Qualitätsmaschine (KEIN Confidence-Score): trennt IDENTIFIZIERT von BEWERTET und begründet jede Lücke. Keine Prozentzahl — auditierbar und ehrlich: „Wir zeigen auch, was wir noch nicht wissen und warum."_')
|
||||
w("")
|
||||
w("**%s**" % rep.completeness_summary)
|
||||
w("")
|
||||
w("> %s" % rep.audit_statement)
|
||||
w("")
|
||||
w("- **Bewertet:** %s (%d Pflichten)" % (", ".join(rep.assessed_regulations), rep.assessed_obligations))
|
||||
w("- **Offen (jeweils begründet):**")
|
||||
for e in rep.exclusions:
|
||||
dq = (" → Rückfrage: `%s`" % e.deciding_question) if e.deciding_question else ""
|
||||
w(" - `%s` — %s `[%s]`%s" % (e.subject, e.reason, e.resolution, dq))
|
||||
w("- **Annahmen:** %s" % ", ".join("%s=%s" % (a.key, a.value) for a in rep.assumptions))
|
||||
w("")
|
||||
w("_Sobald der Umwelt-Korpus (ISO 14001 etc.) landet, kippt `Environmental` automatisch von offen auf bewertet — die Completeness Engine dokumentiert den Fortschritt je Domäne._")
|
||||
w("")
|
||||
coverage_table([
|
||||
("Regulatory Completeness (auditierbar)", "PASS", rep.completeness_summary),
|
||||
("Begründete Ausschlüsse (Korpus/Anwendbarkeit)", "PASS", "%d Ausschlüsse, alle mit Grund" % len(rep.exclusions)),
|
||||
("Fortschritts-Doku je Domäne", "PASS", "Environmental offen→validated bei Korpus-Landung"),
|
||||
])
|
||||
|
||||
@@ -46,7 +46,7 @@ import yaml
|
||||
|
||||
from _helpers import ( # noqa: E402 (script-dir module; keeps generate.py under the LOC budget)
|
||||
OUT, ROLLUP, Row, w, coverage_table, reg_map_block, unsupported_block, interp_status,
|
||||
knowledge_intake_section,
|
||||
knowledge_intake_section, completeness_section,
|
||||
)
|
||||
|
||||
ISO_MAP = {"ISO27001": CapabilityMappingEntry(
|
||||
@@ -465,6 +465,7 @@ coverage_table([
|
||||
])
|
||||
|
||||
knowledge_intake_section(os.path.dirname(__file__)) # Knowledge Intake (impact triage) — kept in _helpers for LOC
|
||||
completeness_section() # Regulatory Completeness — kept in _helpers for LOC
|
||||
|
||||
# ── Epics + roll-up ───────────────────────────────────────────────────────
|
||||
w("## Gaps → Epics (Backlog — nur erfasst, NICHT implementiert)")
|
||||
|
||||
@@ -340,6 +340,31 @@ _So entsteht bei jedem neuen Dokument eine Impact-Analyse statt „200 Seiten PD
|
||||
| Impact-Triage (HIGH/LOW/NONE/new_domain) | **PASS** | 3 Beispiel-Dokumente korrekt eingeordnet |
|
||||
| Regelwerk-ID-Normalisierung | **TODO** | CRA vs Cyber Resilience Act vereinheitlichen |
|
||||
|
||||
## Regulatory Completeness — was wir bewerten konnten, und was bewusst nicht
|
||||
|
||||
_Interne Qualitätsmaschine (KEIN Confidence-Score): trennt IDENTIFIZIERT von BEWERTET und begründet jede Lücke. Keine Prozentzahl — auditierbar und ehrlich: „Wir zeigen auch, was wir noch nicht wissen und warum."_
|
||||
|
||||
**Identifiziert 5 · bewertet 2 · offen 3 · Unsicherheiten 1 · Begründung ja**
|
||||
|
||||
> Für dieses Produkt konnten wir 2 von 5 identifizierten regulatorischen Domänen vollständig bewerten. 3 weitere sind noch nicht Bestandteil des validierten Korpus bzw. anwendungsunsicher und wurden deshalb bewusst nicht bewertet.
|
||||
|
||||
- **Bewertet:** CRA, MaschinenVO (128 Pflichten)
|
||||
- **Offen (jeweils begründet):**
|
||||
- `DataAct` — generates_usage_data = unbekannt `[query_required]` → Rückfrage: `generates_usage_data`
|
||||
- `EMV` — nicht im validierten Korpus `[future_corpus]`
|
||||
- `Environmental` — nicht im validierten Korpus `[future_corpus]`
|
||||
- **Annahmen:** Funkmodul=nein, personenbezogene Nutzungsdaten=nein
|
||||
|
||||
_Sobald der Umwelt-Korpus (ISO 14001 etc.) landet, kippt `Environmental` automatisch von offen auf bewertet — die Completeness Engine dokumentiert den Fortschritt je Domäne._
|
||||
|
||||
**Architecture Coverage**
|
||||
|
||||
| Layer | Status | Hinweis |
|
||||
|---|---|---|
|
||||
| Regulatory Completeness (auditierbar) | **PASS** | Identifiziert 5 · bewertet 2 · offen 3 · Unsicherheiten 1 · Begründung ja |
|
||||
| Begründete Ausschlüsse (Korpus/Anwendbarkeit) | **PASS** | 3 Ausschlüsse, alle mit Grund |
|
||||
| Fortschritts-Doku je Domäne | **PASS** | Environmental offen→validated bei Korpus-Landung |
|
||||
|
||||
## Gaps → Epics (Backlog — nur erfasst, NICHT implementiert)
|
||||
|
||||
| Epic | Titel | schliesst Coverage-Luecke |
|
||||
@@ -351,6 +376,6 @@ _So entsteht bei jedem neuen Dokument eine Impact-Analyse statt „200 Seiten PD
|
||||
|
||||
## Suite-Status (Roll-up)
|
||||
|
||||
- Coverage-Zellen gesamt: **41**
|
||||
- PASS: **30** · PARTIAL: 3 · UNSUPPORTED: 1 · TODO: 6 · N/A: 1 · NEEDS_FACTS: 0
|
||||
- Coverage-Zellen gesamt: **44**
|
||||
- PASS: **33** · PARTIAL: 3 · UNSUPPORTED: 1 · TODO: 6 · N/A: 1 · NEEDS_FACTS: 0
|
||||
- Fortschritt = PASS-Anteil steigt, wenn Epics RS-001…004 landen (objektiver Maßstab, kein LOC).
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Tests for the Regulatory Completeness Engine — auditable coverage, not confidence.
|
||||
|
||||
Acceptance: separate identified from assessed regulations; justify every gap (corpus gap ->
|
||||
future_corpus, applicability uncertain -> query_required with a deciding question); report counts
|
||||
(never a single percentage); emit an honest audit statement. The product shows what it does NOT
|
||||
know and why.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from compliance.completeness import CompletenessReport, CorpusStatus, assess_completeness
|
||||
|
||||
IDENTIFIED = ["CRA", "MaschinenVO", "EMV", "Environmental", "DataAct"]
|
||||
CORPUS = {"CRA": "validated", "MaschinenVO": "validated", "EMV": "validated",
|
||||
"Environmental": "unsupported", "DataAct": "validated"}
|
||||
UNCERTAIN = [{"regulation": "DataAct", "deciding_question": "generates_usage_data", "reason": "generates_usage_data unbekannt"}]
|
||||
|
||||
|
||||
def _report():
|
||||
return assess_completeness(IDENTIFIED, CORPUS, uncertain=UNCERTAIN,
|
||||
assumptions=[{"key": "funkmodul", "value": "nein"}], assessed_obligations=128)
|
||||
|
||||
|
||||
def test_assessed_excludes_uncertain_even_if_corpus_validated():
|
||||
r = _report()
|
||||
# DataAct has a validated corpus but uncertain applicability -> NOT assessed
|
||||
assert r.assessed_regulations == ["CRA", "EMV", "MaschinenVO"]
|
||||
assert "DataAct" in r.open_regulations and "Environmental" in r.open_regulations
|
||||
|
||||
|
||||
def test_corpus_gap_vs_applicability_exclusion():
|
||||
r = _report()
|
||||
by = {e.subject: e for e in r.exclusions}
|
||||
assert by["DataAct"].resolution == "query_required" and by["DataAct"].deciding_question == "generates_usage_data"
|
||||
assert by["Environmental"].resolution == "future_corpus"
|
||||
|
||||
|
||||
def test_open_corpora_is_unsupported_only():
|
||||
r = _report()
|
||||
# DataAct corpus is validated (only applicability is open) -> NOT an open corpus
|
||||
assert r.open_corpora == ["Environmental"]
|
||||
|
||||
|
||||
def test_justification_present_when_every_gap_has_a_reason():
|
||||
r = _report()
|
||||
assert r.justification_present is True
|
||||
open_subjects = {e.subject for e in r.exclusions}
|
||||
assert set(r.open_regulations) <= open_subjects
|
||||
|
||||
|
||||
def test_counts_summary_has_no_percentage():
|
||||
r = _report()
|
||||
assert "%" not in r.completeness_summary and "%" not in r.audit_statement
|
||||
assert "Identifiziert 5" in r.completeness_summary and "bewertet 3" in r.completeness_summary
|
||||
|
||||
|
||||
def test_audit_statement_is_honest():
|
||||
r = _report()
|
||||
assert "3 von 5" in r.audit_statement and "bewusst nicht bewertet" in r.audit_statement
|
||||
|
||||
|
||||
def test_draft_corpus_is_in_review_exclusion():
|
||||
r = assess_completeness(["CRA", "IEC62443"], {"CRA": "validated", "IEC62443": "draft"})
|
||||
by = {e.subject: e for e in r.exclusions}
|
||||
assert by["IEC62443"].resolution == "in_review"
|
||||
assert "IEC62443" in r.open_regulations and "IEC62443" not in r.open_corpora # draft != missing corpus
|
||||
|
||||
|
||||
def test_all_assessed_no_open():
|
||||
r = assess_completeness(["CRA", "MaschinenVO"], {"CRA": "validated", "MaschinenVO": "validated"})
|
||||
assert r.open_regulations == [] and r.exclusions == []
|
||||
assert r.justification_present is True
|
||||
assert "alle 2" in r.audit_statement
|
||||
|
||||
|
||||
def test_coverage_status_mapped():
|
||||
r = _report()
|
||||
cov = {c.regulation: c.status for c in r.coverage}
|
||||
assert cov["CRA"] == CorpusStatus.VALIDATED and cov["Environmental"] == CorpusStatus.UNSUPPORTED
|
||||
|
||||
|
||||
def test_assumptions_and_obligations_carried():
|
||||
r = _report()
|
||||
assert r.assessed_obligations == 128
|
||||
assert [a.key for a in r.assumptions] == ["funkmodul"]
|
||||
|
||||
|
||||
def test_unknown_regulation_defaults_to_open_corpus():
|
||||
r = assess_completeness(["CRA", "REACH"], {"CRA": "validated"}) # REACH not in registry -> unknown
|
||||
assert "REACH" in r.open_corpora and r.assessed_regulations == ["CRA"]
|
||||
|
||||
|
||||
def test_deterministic_and_type():
|
||||
r1 = _report()
|
||||
r2 = _report()
|
||||
assert r1.model_dump() == r2.model_dump()
|
||||
assert isinstance(r1, CompletenessReport)
|
||||
Reference in New Issue
Block a user