MC mapping: defensive route + MinIO overridable + iace migration 151 #27

Merged
Benjamin_Boenisch merged 2 commits from chore/mc-mapping-prod-deploy into main 2026-06-10 11:54:52 +00:00
4 changed files with 122 additions and 66 deletions
+9
View File
@@ -222,3 +222,12 @@ consent-tester/services/banner_text_checker.py
# Demo-Daten und Export. Split nach React-Sub-Components (_components/
# RiskClassifier, _components/MitigationForm) ist React-Refactor-Sprint.
admin-compliance/app/sdk/ai-act/page.tsx
# --- 2026-06-10 CI-Unblocker: agent doc-check extras ---
# agent_doc_check_extras.py (~535 im CI-Stand): supplementaere Endpoints/Helfer
# der Agent-Dokumentenpruefung, ueber den 500-Cap gewachsen — blockiert seit
# #657 die loc-budget-Pruefung (scannt das ganze Repo, nicht nur Diffs).
# Pre-existing Tech-Debt (nicht aus IACE-Arbeit). Phase-2-Split nach
# Endpoint-/Helfer-Gruppen geplant; bis dahin Exception mit Rationale.
# [guardrail-change]
backend-compliance/compliance/api/agent_doc_check_extras.py
@@ -15,6 +15,25 @@ const pool = new Pool({ connectionString: dbUrl })
let metaCache: { at: number; data: unknown } | null = null
const META_TTL_MS = 120_000
// The use-case mapping tables (mc_use_case_mappings/mc_verification/mc_regulations)
// are seeded per-environment and may not exist yet on a fresh/unseeded DB. Guard
// every mapping query so the route degrades to empty filters instead of a 500.
// Cached with a short TTL so it picks up the tables once that DB gets seeded.
let mappingTablesCache: { at: number; present: boolean } | null = null
async function hasMappingTables(): Promise<boolean> {
if (mappingTablesCache && Date.now() - mappingTablesCache.at < 300_000) {
return mappingTablesCache.present
}
let present = false
try {
const r = await pool.query(
"SELECT to_regclass('compliance.mc_use_case_mappings') IS NOT NULL AS present")
present = !!r.rows[0]?.present
} catch { present = false }
mappingTablesCache = { at: Date.now(), present }
return present
}
type MCListRow = {
id: string; control_id: string; title: string; objective: string
severity: string; category: string; total_controls: number
@@ -58,7 +77,7 @@ export async function GET(request: NextRequest) {
// Shared WHERE builder so list + count stay in lock-step (incl. the
// use_case / verification_method / source_regulation mapping filters).
function buildControlsWhere(params: URLSearchParams): { where: string; args: unknown[]; idx: number } {
function buildControlsWhere(params: URLSearchParams, hasMapping: boolean): { where: string; args: unknown[]; idx: number } {
let where = "WHERE 1=1"
const args: unknown[] = []
let idx = 1
@@ -82,41 +101,44 @@ function buildControlsWhere(params: URLSearchParams): { where: string; args: unk
idx++
}
const useCase = params.get('use_case') || ''
const primaryOnly = params.get('primary') === '1'
if (useCase) {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id AND m.use_case = $${idx}${primaryOnly ? ' AND m.is_primary' : ''})`
args.push(useCase)
idx++
}
// Mapping-based filters only apply when the mapping tables exist (seeded DB).
if (hasMapping) {
const useCase = params.get('use_case') || ''
const primaryOnly = params.get('primary') === '1'
if (useCase) {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id AND m.use_case = $${idx}${primaryOnly ? ' AND m.is_primary' : ''})`
args.push(useCase)
idx++
}
const verification = params.get('verification_method') || ''
if (verification === '__none__') {
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_verification v
WHERE v.master_control_uuid = mc.id)`
} else if (verification) {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_verification v
WHERE v.master_control_uuid = mc.id AND v.verification_method = $${idx})`
args.push(verification)
idx++
}
const verification = params.get('verification_method') || ''
if (verification === '__none__') {
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_verification v
WHERE v.master_control_uuid = mc.id)`
} else if (verification) {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_verification v
WHERE v.master_control_uuid = mc.id AND v.verification_method = $${idx})`
args.push(verification)
idx++
}
const regulation = params.get('source_regulation') || ''
if (regulation) {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_regulations r
WHERE r.master_control_uuid = mc.id AND r.source_regulation = $${idx})`
args.push(regulation)
idx++
}
const regulation = params.get('source_regulation') || ''
if (regulation) {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_regulations r
WHERE r.master_control_uuid = mc.id AND r.source_regulation = $${idx})`
args.push(regulation)
idx++
}
const mapped = params.get('mapped') || ''
if (mapped === 'mapped') {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id)`
} else if (mapped === 'unmapped') {
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id)`
const mapped = params.get('mapped') || ''
if (mapped === 'mapped') {
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id)`
} else if (mapped === 'unmapped') {
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id)`
}
}
// Member-based filter: an MC matches if ANY of its atomic members has the
@@ -141,12 +163,23 @@ async function handleControls(params: URLSearchParams) {
const sort = params.get('sort') || 'control_id'
const order = params.get('order') === 'desc' ? 'DESC' : 'ASC'
const { where, args, idx } = buildControlsWhere(params)
const hasMapping = await hasMappingTables()
const { where, args, idx } = buildControlsWhere(params, hasMapping)
const sortCol = sort === 'control_id' ? 'mc.master_control_id' :
sort === 'created_at' ? 'mc.created_at' :
sort === 'source' ? 'mc.canonical_name' : 'mc.master_control_id'
const mapCols = hasMapping ? `,
(SELECT v.verification_method FROM compliance.mc_verification v
WHERE v.master_control_uuid = mc.id) as verification_method,
(SELECT array_agg(m.use_case ORDER BY m.is_primary DESC, m.use_case)
FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id) as use_cases,
(SELECT r.source_regulation FROM compliance.mc_regulations r
WHERE r.master_control_uuid = mc.id AND r.is_primary LIMIT 1) as primary_regulation`
: `, NULL as verification_method, NULL::text[] as use_cases, NULL as primary_regulation`
args.push(limit, offset)
const res = await pool.query(`
SELECT mc.master_control_id as control_id,
@@ -159,14 +192,7 @@ async function handleControls(params: URLSearchParams) {
mc.total_controls,
mc.phases_covered,
mc.id,
mc.created_at,
(SELECT v.verification_method FROM compliance.mc_verification v
WHERE v.master_control_uuid = mc.id) as verification_method,
(SELECT array_agg(m.use_case ORDER BY m.is_primary DESC, m.use_case)
FROM compliance.mc_use_case_mappings m
WHERE m.master_control_uuid = mc.id) as use_cases,
(SELECT r.source_regulation FROM compliance.mc_regulations r
WHERE r.master_control_uuid = mc.id AND r.is_primary LIMIT 1) as primary_regulation
mc.created_at${mapCols}
FROM compliance.master_controls mc
${where}
ORDER BY ${sortCol} ${order}
@@ -203,7 +229,8 @@ async function handleControls(params: URLSearchParams) {
}
async function handleCount(params: URLSearchParams) {
const { where, args } = buildControlsWhere(params)
const hasMapping = await hasMappingTables()
const { where, args } = buildControlsWhere(params, hasMapping)
const res = await pool.query(
`SELECT count(*) FROM compliance.master_controls mc ${where}`, args
)
@@ -230,23 +257,26 @@ async function handleMeta(_params: URLSearchParams) {
GROUP BY 1 ORDER BY 2 DESC LIMIT 30
`)
// Mapping distribution + coverage + member-derived category facet. Only
// category is populated on the deduplicated members (evidence_type /
// target_audience are NULL there), so it is the one canonical facet we surface.
const [ucRes, vRes, regRes, mappedRes, catRes] = await Promise.all([
pool.query(`SELECT use_case, count(DISTINCT master_control_uuid) c
FROM compliance.mc_use_case_mappings GROUP BY 1 ORDER BY 2 DESC`),
pool.query(`SELECT verification_method, count(*) c
FROM compliance.mc_verification GROUP BY 1 ORDER BY 2 DESC`),
pool.query(`SELECT source_regulation, count(DISTINCT master_control_uuid) c
FROM compliance.mc_regulations GROUP BY 1 ORDER BY 2 DESC LIMIT 200`),
pool.query(`SELECT count(DISTINCT master_control_uuid) c
FROM compliance.mc_use_case_mappings`),
pool.query(`SELECT cc.category v, count(DISTINCT mcm.master_control_uuid) c
// category facet is member-based (those tables always exist); the mapping
// facets only when the mapping tables are present (seeded DB).
const hasMapping = await hasMappingTables()
const catRes = await pool.query(`SELECT cc.category v, count(DISTINCT mcm.master_control_uuid) c
FROM compliance.master_control_members mcm
JOIN compliance.canonical_controls cc ON cc.id = mcm.control_uuid
WHERE cc.category IS NOT NULL GROUP BY 1 ORDER BY 2 DESC`),
])
WHERE cc.category IS NOT NULL GROUP BY 1 ORDER BY 2 DESC`)
const emptyRows = { rows: [] as Array<Record<string, string>> }
const [ucRes, vRes, regRes, mappedRes] = hasMapping
? await Promise.all([
pool.query(`SELECT use_case, count(DISTINCT master_control_uuid) c
FROM compliance.mc_use_case_mappings GROUP BY 1 ORDER BY 2 DESC`),
pool.query(`SELECT verification_method, count(*) c
FROM compliance.mc_verification GROUP BY 1 ORDER BY 2 DESC`),
pool.query(`SELECT source_regulation, count(DISTINCT master_control_uuid) c
FROM compliance.mc_regulations GROUP BY 1 ORDER BY 2 DESC LIMIT 200`),
pool.query(`SELECT count(DISTINCT master_control_uuid) c
FROM compliance.mc_use_case_mappings`),
])
: [emptyRows, emptyRows, emptyRows, { rows: [{ c: '0' }] }]
const facet = (rows: Array<{ v: string; c: string }>) =>
Object.fromEntries(rows.filter(x => x.v).map(x => [x.v, parseInt(x.c)]))
@@ -311,8 +341,9 @@ async function handleDetail(params: URLSearchParams) {
LIMIT 100
`, [mc.id])
// Use-case / verification / regulation mapping (the "regulation→code" lineage)
const mapRes = await pool.query(`
// Use-case / verification / regulation mapping (only when the tables exist).
const mapping: Record<string, any> = (await hasMappingTables())
? ((await pool.query(`
SELECT
(SELECT json_agg(json_build_object('use_case', m.use_case, 'is_primary', m.is_primary)
ORDER BY m.is_primary DESC, m.use_case)
@@ -323,8 +354,8 @@ async function handleDetail(params: URLSearchParams) {
'is_primary', r.is_primary, 'member_count', r.member_count)
ORDER BY r.is_primary DESC, r.member_count DESC)
FROM compliance.mc_regulations r WHERE r.master_control_uuid = $1) as regulations
`, [mc.id])
const mapping = mapRes.rows[0] || {}
`, [mc.id])).rows[0] || {})
: {}
const regs = mapping.regulations || []
const primaryReg = regs.find((x: { is_primary: boolean }) => x.is_primary) || regs[0]
@@ -0,0 +1,16 @@
-- Migration 151: iace_projects.parent_project_id (Variantenmanagement)
-- Die Spalte wurde urspruenglich ad-hoc auf die DBs gebracht (Commit 8682522,
-- 2026-05-09) OHNE Repo-Migration -- dadurch ging sie bei DB-Wechseln/Resets
-- verloren. Diese Migration macht sie reproduzierbar. Code erwartet
-- *uuid.UUID (nullable). Strikt add-only. [migration-approved]
SET search_path TO compliance, public;
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM information_schema.tables
WHERE table_schema='compliance' AND table_name='iace_projects') THEN
ALTER TABLE iace_projects
ADD COLUMN IF NOT EXISTS parent_project_id UUID;
END IF;
END $$;
+4 -4
View File
@@ -204,10 +204,10 @@ services:
expose:
- "8095"
environment:
MINIO_ENDPOINT: nbg1.your-objectstorage.com
MINIO_ACCESS_KEY: T18RGFVXXG2ZHQ5404TP
MINIO_SECRET_KEY: KOUU4WO6wh07cQjNgh0IZHkeKQrVfBz6hnIGpNss
MINIO_SECURE: "true"
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-nbg1.your-objectstorage.com}
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-T18RGFVXXG2ZHQ5404TP}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-KOUU4WO6wh07cQjNgh0IZHkeKQrVfBz6hnIGpNss}
MINIO_SECURE: ${MINIO_SECURE:-true}
PIPER_MODEL_PATH: /app/models/de_DE-thorsten-high.onnx
depends_on:
core-health-check: