From bb6139df3e4eb2571561110e4914135bdc385062 Mon Sep 17 00:00:00 2001 From: Benjamin_Boenisch Date: Wed, 10 Jun 2026 11:54:48 +0000 Subject: [PATCH] MC mapping: defensive route + MinIO overridable + iace migration 151 (#27) MC mapping deploy: defensive route + MinIO overridable + Migration 151 + loc-exception [migration-approved] [guardrail-change] --- .claude/rules/loc-exceptions.txt | 9 + .../app/api/sdk/v1/master-controls/route.ts | 155 +++++++++++------- .../migrations/151_iace_parent_project_id.sql | 16 ++ docker-compose.yml | 8 +- 4 files changed, 122 insertions(+), 66 deletions(-) create mode 100644 backend-compliance/migrations/151_iace_parent_project_id.sql diff --git a/.claude/rules/loc-exceptions.txt b/.claude/rules/loc-exceptions.txt index 0264aebb..853cb6ba 100644 --- a/.claude/rules/loc-exceptions.txt +++ b/.claude/rules/loc-exceptions.txt @@ -222,3 +222,12 @@ consent-tester/services/banner_text_checker.py # Demo-Daten und Export. Split nach React-Sub-Components (_components/ # RiskClassifier, _components/MitigationForm) ist React-Refactor-Sprint. admin-compliance/app/sdk/ai-act/page.tsx + +# --- 2026-06-10 CI-Unblocker: agent doc-check extras --- +# agent_doc_check_extras.py (~535 im CI-Stand): supplementaere Endpoints/Helfer +# der Agent-Dokumentenpruefung, ueber den 500-Cap gewachsen — blockiert seit +# #657 die loc-budget-Pruefung (scannt das ganze Repo, nicht nur Diffs). +# Pre-existing Tech-Debt (nicht aus IACE-Arbeit). Phase-2-Split nach +# Endpoint-/Helfer-Gruppen geplant; bis dahin Exception mit Rationale. +# [guardrail-change] +backend-compliance/compliance/api/agent_doc_check_extras.py diff --git a/admin-compliance/app/api/sdk/v1/master-controls/route.ts b/admin-compliance/app/api/sdk/v1/master-controls/route.ts index 009e14a9..6c443e29 100644 --- a/admin-compliance/app/api/sdk/v1/master-controls/route.ts +++ b/admin-compliance/app/api/sdk/v1/master-controls/route.ts @@ -15,6 +15,25 @@ const pool = new Pool({ connectionString: dbUrl }) let metaCache: { at: number; data: unknown } | null = null const META_TTL_MS = 120_000 +// The use-case mapping tables (mc_use_case_mappings/mc_verification/mc_regulations) +// are seeded per-environment and may not exist yet on a fresh/unseeded DB. Guard +// every mapping query so the route degrades to empty filters instead of a 500. +// Cached with a short TTL so it picks up the tables once that DB gets seeded. +let mappingTablesCache: { at: number; present: boolean } | null = null +async function hasMappingTables(): Promise { + if (mappingTablesCache && Date.now() - mappingTablesCache.at < 300_000) { + return mappingTablesCache.present + } + let present = false + try { + const r = await pool.query( + "SELECT to_regclass('compliance.mc_use_case_mappings') IS NOT NULL AS present") + present = !!r.rows[0]?.present + } catch { present = false } + mappingTablesCache = { at: Date.now(), present } + return present +} + type MCListRow = { id: string; control_id: string; title: string; objective: string severity: string; category: string; total_controls: number @@ -58,7 +77,7 @@ export async function GET(request: NextRequest) { // Shared WHERE builder so list + count stay in lock-step (incl. the // use_case / verification_method / source_regulation mapping filters). -function buildControlsWhere(params: URLSearchParams): { where: string; args: unknown[]; idx: number } { +function buildControlsWhere(params: URLSearchParams, hasMapping: boolean): { where: string; args: unknown[]; idx: number } { let where = "WHERE 1=1" const args: unknown[] = [] let idx = 1 @@ -82,41 +101,44 @@ function buildControlsWhere(params: URLSearchParams): { where: string; args: unk idx++ } - const useCase = params.get('use_case') || '' - const primaryOnly = params.get('primary') === '1' - if (useCase) { - where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m - WHERE m.master_control_uuid = mc.id AND m.use_case = $${idx}${primaryOnly ? ' AND m.is_primary' : ''})` - args.push(useCase) - idx++ - } + // Mapping-based filters only apply when the mapping tables exist (seeded DB). + if (hasMapping) { + const useCase = params.get('use_case') || '' + const primaryOnly = params.get('primary') === '1' + if (useCase) { + where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m + WHERE m.master_control_uuid = mc.id AND m.use_case = $${idx}${primaryOnly ? ' AND m.is_primary' : ''})` + args.push(useCase) + idx++ + } - const verification = params.get('verification_method') || '' - if (verification === '__none__') { - where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_verification v - WHERE v.master_control_uuid = mc.id)` - } else if (verification) { - where += ` AND EXISTS (SELECT 1 FROM compliance.mc_verification v - WHERE v.master_control_uuid = mc.id AND v.verification_method = $${idx})` - args.push(verification) - idx++ - } + const verification = params.get('verification_method') || '' + if (verification === '__none__') { + where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_verification v + WHERE v.master_control_uuid = mc.id)` + } else if (verification) { + where += ` AND EXISTS (SELECT 1 FROM compliance.mc_verification v + WHERE v.master_control_uuid = mc.id AND v.verification_method = $${idx})` + args.push(verification) + idx++ + } - const regulation = params.get('source_regulation') || '' - if (regulation) { - where += ` AND EXISTS (SELECT 1 FROM compliance.mc_regulations r - WHERE r.master_control_uuid = mc.id AND r.source_regulation = $${idx})` - args.push(regulation) - idx++ - } + const regulation = params.get('source_regulation') || '' + if (regulation) { + where += ` AND EXISTS (SELECT 1 FROM compliance.mc_regulations r + WHERE r.master_control_uuid = mc.id AND r.source_regulation = $${idx})` + args.push(regulation) + idx++ + } - const mapped = params.get('mapped') || '' - if (mapped === 'mapped') { - where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m - WHERE m.master_control_uuid = mc.id)` - } else if (mapped === 'unmapped') { - where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m - WHERE m.master_control_uuid = mc.id)` + const mapped = params.get('mapped') || '' + if (mapped === 'mapped') { + where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m + WHERE m.master_control_uuid = mc.id)` + } else if (mapped === 'unmapped') { + where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m + WHERE m.master_control_uuid = mc.id)` + } } // Member-based filter: an MC matches if ANY of its atomic members has the @@ -141,12 +163,23 @@ async function handleControls(params: URLSearchParams) { const sort = params.get('sort') || 'control_id' const order = params.get('order') === 'desc' ? 'DESC' : 'ASC' - const { where, args, idx } = buildControlsWhere(params) + const hasMapping = await hasMappingTables() + const { where, args, idx } = buildControlsWhere(params, hasMapping) const sortCol = sort === 'control_id' ? 'mc.master_control_id' : sort === 'created_at' ? 'mc.created_at' : sort === 'source' ? 'mc.canonical_name' : 'mc.master_control_id' + const mapCols = hasMapping ? `, + (SELECT v.verification_method FROM compliance.mc_verification v + WHERE v.master_control_uuid = mc.id) as verification_method, + (SELECT array_agg(m.use_case ORDER BY m.is_primary DESC, m.use_case) + FROM compliance.mc_use_case_mappings m + WHERE m.master_control_uuid = mc.id) as use_cases, + (SELECT r.source_regulation FROM compliance.mc_regulations r + WHERE r.master_control_uuid = mc.id AND r.is_primary LIMIT 1) as primary_regulation` + : `, NULL as verification_method, NULL::text[] as use_cases, NULL as primary_regulation` + args.push(limit, offset) const res = await pool.query(` SELECT mc.master_control_id as control_id, @@ -159,14 +192,7 @@ async function handleControls(params: URLSearchParams) { mc.total_controls, mc.phases_covered, mc.id, - mc.created_at, - (SELECT v.verification_method FROM compliance.mc_verification v - WHERE v.master_control_uuid = mc.id) as verification_method, - (SELECT array_agg(m.use_case ORDER BY m.is_primary DESC, m.use_case) - FROM compliance.mc_use_case_mappings m - WHERE m.master_control_uuid = mc.id) as use_cases, - (SELECT r.source_regulation FROM compliance.mc_regulations r - WHERE r.master_control_uuid = mc.id AND r.is_primary LIMIT 1) as primary_regulation + mc.created_at${mapCols} FROM compliance.master_controls mc ${where} ORDER BY ${sortCol} ${order} @@ -203,7 +229,8 @@ async function handleControls(params: URLSearchParams) { } async function handleCount(params: URLSearchParams) { - const { where, args } = buildControlsWhere(params) + const hasMapping = await hasMappingTables() + const { where, args } = buildControlsWhere(params, hasMapping) const res = await pool.query( `SELECT count(*) FROM compliance.master_controls mc ${where}`, args ) @@ -230,23 +257,26 @@ async function handleMeta(_params: URLSearchParams) { GROUP BY 1 ORDER BY 2 DESC LIMIT 30 `) - // Mapping distribution + coverage + member-derived category facet. Only - // category is populated on the deduplicated members (evidence_type / - // target_audience are NULL there), so it is the one canonical facet we surface. - const [ucRes, vRes, regRes, mappedRes, catRes] = await Promise.all([ - pool.query(`SELECT use_case, count(DISTINCT master_control_uuid) c - FROM compliance.mc_use_case_mappings GROUP BY 1 ORDER BY 2 DESC`), - pool.query(`SELECT verification_method, count(*) c - FROM compliance.mc_verification GROUP BY 1 ORDER BY 2 DESC`), - pool.query(`SELECT source_regulation, count(DISTINCT master_control_uuid) c - FROM compliance.mc_regulations GROUP BY 1 ORDER BY 2 DESC LIMIT 200`), - pool.query(`SELECT count(DISTINCT master_control_uuid) c - FROM compliance.mc_use_case_mappings`), - pool.query(`SELECT cc.category v, count(DISTINCT mcm.master_control_uuid) c + // category facet is member-based (those tables always exist); the mapping + // facets only when the mapping tables are present (seeded DB). + const hasMapping = await hasMappingTables() + const catRes = await pool.query(`SELECT cc.category v, count(DISTINCT mcm.master_control_uuid) c FROM compliance.master_control_members mcm JOIN compliance.canonical_controls cc ON cc.id = mcm.control_uuid - WHERE cc.category IS NOT NULL GROUP BY 1 ORDER BY 2 DESC`), - ]) + WHERE cc.category IS NOT NULL GROUP BY 1 ORDER BY 2 DESC`) + const emptyRows = { rows: [] as Array> } + const [ucRes, vRes, regRes, mappedRes] = hasMapping + ? await Promise.all([ + pool.query(`SELECT use_case, count(DISTINCT master_control_uuid) c + FROM compliance.mc_use_case_mappings GROUP BY 1 ORDER BY 2 DESC`), + pool.query(`SELECT verification_method, count(*) c + FROM compliance.mc_verification GROUP BY 1 ORDER BY 2 DESC`), + pool.query(`SELECT source_regulation, count(DISTINCT master_control_uuid) c + FROM compliance.mc_regulations GROUP BY 1 ORDER BY 2 DESC LIMIT 200`), + pool.query(`SELECT count(DISTINCT master_control_uuid) c + FROM compliance.mc_use_case_mappings`), + ]) + : [emptyRows, emptyRows, emptyRows, { rows: [{ c: '0' }] }] const facet = (rows: Array<{ v: string; c: string }>) => Object.fromEntries(rows.filter(x => x.v).map(x => [x.v, parseInt(x.c)])) @@ -311,8 +341,9 @@ async function handleDetail(params: URLSearchParams) { LIMIT 100 `, [mc.id]) - // Use-case / verification / regulation mapping (the "regulation→code" lineage) - const mapRes = await pool.query(` + // Use-case / verification / regulation mapping (only when the tables exist). + const mapping: Record = (await hasMappingTables()) + ? ((await pool.query(` SELECT (SELECT json_agg(json_build_object('use_case', m.use_case, 'is_primary', m.is_primary) ORDER BY m.is_primary DESC, m.use_case) @@ -323,8 +354,8 @@ async function handleDetail(params: URLSearchParams) { 'is_primary', r.is_primary, 'member_count', r.member_count) ORDER BY r.is_primary DESC, r.member_count DESC) FROM compliance.mc_regulations r WHERE r.master_control_uuid = $1) as regulations - `, [mc.id]) - const mapping = mapRes.rows[0] || {} + `, [mc.id])).rows[0] || {}) + : {} const regs = mapping.regulations || [] const primaryReg = regs.find((x: { is_primary: boolean }) => x.is_primary) || regs[0] diff --git a/backend-compliance/migrations/151_iace_parent_project_id.sql b/backend-compliance/migrations/151_iace_parent_project_id.sql new file mode 100644 index 00000000..d44056c6 --- /dev/null +++ b/backend-compliance/migrations/151_iace_parent_project_id.sql @@ -0,0 +1,16 @@ +-- Migration 151: iace_projects.parent_project_id (Variantenmanagement) +-- Die Spalte wurde urspruenglich ad-hoc auf die DBs gebracht (Commit 8682522, +-- 2026-05-09) OHNE Repo-Migration -- dadurch ging sie bei DB-Wechseln/Resets +-- verloren. Diese Migration macht sie reproduzierbar. Code erwartet +-- *uuid.UUID (nullable). Strikt add-only. [migration-approved] + +SET search_path TO compliance, public; + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.tables + WHERE table_schema='compliance' AND table_name='iace_projects') THEN + ALTER TABLE iace_projects + ADD COLUMN IF NOT EXISTS parent_project_id UUID; + END IF; +END $$; diff --git a/docker-compose.yml b/docker-compose.yml index c6812c1f..a8c8449f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -204,10 +204,10 @@ services: expose: - "8095" environment: - MINIO_ENDPOINT: nbg1.your-objectstorage.com - MINIO_ACCESS_KEY: T18RGFVXXG2ZHQ5404TP - MINIO_SECRET_KEY: KOUU4WO6wh07cQjNgh0IZHkeKQrVfBz6hnIGpNss - MINIO_SECURE: "true" + MINIO_ENDPOINT: ${MINIO_ENDPOINT:-nbg1.your-objectstorage.com} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-T18RGFVXXG2ZHQ5404TP} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-KOUU4WO6wh07cQjNgh0IZHkeKQrVfBz6hnIGpNss} + MINIO_SECURE: ${MINIO_SECURE:-true} PIPER_MODEL_PATH: /app/models/de_DE-thorsten-high.onnx depends_on: core-health-check: