fix(mc): defensive mapping queries + MinIO env overridable + iace migration 151
CI / detect-changes (pull_request) Failing after 6s
CI / branch-name (pull_request) Successful in 1s
CI / guardrail-integrity (pull_request) Failing after 4s
CI / secret-scan (pull_request) Failing after 6s
CI / dep-audit (pull_request) Failing after 12s
CI / sbom-scan (pull_request) Failing after 2s
CI / build-sha-integrity (pull_request) Failing after 4s
CI / validate-canonical-controls (pull_request) Failing after 9s
CI / loc-budget (pull_request) Has been skipped
CI / go-lint (pull_request) Has been skipped
CI / python-lint (pull_request) Has been skipped
CI / nodejs-lint (pull_request) Has been skipped
CI / nodejs-build (pull_request) Has been skipped
CI / test-go (pull_request) Has been skipped
CI / iace-gt-coverage (pull_request) Has been skipped
CI / test-python-backend (pull_request) Has been skipped
CI / test-python-document-crawler (pull_request) Has been skipped
CI / test-python-dsms-gateway (pull_request) Has been skipped
CI / detect-changes (pull_request) Failing after 6s
CI / branch-name (pull_request) Successful in 1s
CI / guardrail-integrity (pull_request) Failing after 4s
CI / secret-scan (pull_request) Failing after 6s
CI / dep-audit (pull_request) Failing after 12s
CI / sbom-scan (pull_request) Failing after 2s
CI / build-sha-integrity (pull_request) Failing after 4s
CI / validate-canonical-controls (pull_request) Failing after 9s
CI / loc-budget (pull_request) Has been skipped
CI / go-lint (pull_request) Has been skipped
CI / python-lint (pull_request) Has been skipped
CI / nodejs-lint (pull_request) Has been skipped
CI / nodejs-build (pull_request) Has been skipped
CI / test-go (pull_request) Has been skipped
CI / iace-gt-coverage (pull_request) Has been skipped
CI / test-python-backend (pull_request) Has been skipped
CI / test-python-document-crawler (pull_request) Has been skipped
CI / test-python-dsms-gateway (pull_request) Has been skipped
- master-controls route: guard all mapping queries with hasMappingTables() so an unseeded DB degrades to empty filters instead of a 500. - docker-compose: MinIO endpoint/keys/secure overridable via env (prod defaults preserved) — enables per-environment local config. - migration 151: reproducible iace_projects.parent_project_id (was ad-hoc). [migration-approved] Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,25 @@ const pool = new Pool({ connectionString: dbUrl })
|
||||
let metaCache: { at: number; data: unknown } | null = null
|
||||
const META_TTL_MS = 120_000
|
||||
|
||||
// The use-case mapping tables (mc_use_case_mappings/mc_verification/mc_regulations)
|
||||
// are seeded per-environment and may not exist yet on a fresh/unseeded DB. Guard
|
||||
// every mapping query so the route degrades to empty filters instead of a 500.
|
||||
// Cached with a short TTL so it picks up the tables once that DB gets seeded.
|
||||
let mappingTablesCache: { at: number; present: boolean } | null = null
|
||||
async function hasMappingTables(): Promise<boolean> {
|
||||
if (mappingTablesCache && Date.now() - mappingTablesCache.at < 300_000) {
|
||||
return mappingTablesCache.present
|
||||
}
|
||||
let present = false
|
||||
try {
|
||||
const r = await pool.query(
|
||||
"SELECT to_regclass('compliance.mc_use_case_mappings') IS NOT NULL AS present")
|
||||
present = !!r.rows[0]?.present
|
||||
} catch { present = false }
|
||||
mappingTablesCache = { at: Date.now(), present }
|
||||
return present
|
||||
}
|
||||
|
||||
type MCListRow = {
|
||||
id: string; control_id: string; title: string; objective: string
|
||||
severity: string; category: string; total_controls: number
|
||||
@@ -58,7 +77,7 @@ export async function GET(request: NextRequest) {
|
||||
|
||||
// Shared WHERE builder so list + count stay in lock-step (incl. the
|
||||
// use_case / verification_method / source_regulation mapping filters).
|
||||
function buildControlsWhere(params: URLSearchParams): { where: string; args: unknown[]; idx: number } {
|
||||
function buildControlsWhere(params: URLSearchParams, hasMapping: boolean): { where: string; args: unknown[]; idx: number } {
|
||||
let where = "WHERE 1=1"
|
||||
const args: unknown[] = []
|
||||
let idx = 1
|
||||
@@ -82,41 +101,44 @@ function buildControlsWhere(params: URLSearchParams): { where: string; args: unk
|
||||
idx++
|
||||
}
|
||||
|
||||
const useCase = params.get('use_case') || ''
|
||||
const primaryOnly = params.get('primary') === '1'
|
||||
if (useCase) {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id AND m.use_case = $${idx}${primaryOnly ? ' AND m.is_primary' : ''})`
|
||||
args.push(useCase)
|
||||
idx++
|
||||
}
|
||||
// Mapping-based filters only apply when the mapping tables exist (seeded DB).
|
||||
if (hasMapping) {
|
||||
const useCase = params.get('use_case') || ''
|
||||
const primaryOnly = params.get('primary') === '1'
|
||||
if (useCase) {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id AND m.use_case = $${idx}${primaryOnly ? ' AND m.is_primary' : ''})`
|
||||
args.push(useCase)
|
||||
idx++
|
||||
}
|
||||
|
||||
const verification = params.get('verification_method') || ''
|
||||
if (verification === '__none__') {
|
||||
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_verification v
|
||||
WHERE v.master_control_uuid = mc.id)`
|
||||
} else if (verification) {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_verification v
|
||||
WHERE v.master_control_uuid = mc.id AND v.verification_method = $${idx})`
|
||||
args.push(verification)
|
||||
idx++
|
||||
}
|
||||
const verification = params.get('verification_method') || ''
|
||||
if (verification === '__none__') {
|
||||
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_verification v
|
||||
WHERE v.master_control_uuid = mc.id)`
|
||||
} else if (verification) {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_verification v
|
||||
WHERE v.master_control_uuid = mc.id AND v.verification_method = $${idx})`
|
||||
args.push(verification)
|
||||
idx++
|
||||
}
|
||||
|
||||
const regulation = params.get('source_regulation') || ''
|
||||
if (regulation) {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_regulations r
|
||||
WHERE r.master_control_uuid = mc.id AND r.source_regulation = $${idx})`
|
||||
args.push(regulation)
|
||||
idx++
|
||||
}
|
||||
const regulation = params.get('source_regulation') || ''
|
||||
if (regulation) {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_regulations r
|
||||
WHERE r.master_control_uuid = mc.id AND r.source_regulation = $${idx})`
|
||||
args.push(regulation)
|
||||
idx++
|
||||
}
|
||||
|
||||
const mapped = params.get('mapped') || ''
|
||||
if (mapped === 'mapped') {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id)`
|
||||
} else if (mapped === 'unmapped') {
|
||||
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id)`
|
||||
const mapped = params.get('mapped') || ''
|
||||
if (mapped === 'mapped') {
|
||||
where += ` AND EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id)`
|
||||
} else if (mapped === 'unmapped') {
|
||||
where += ` AND NOT EXISTS (SELECT 1 FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id)`
|
||||
}
|
||||
}
|
||||
|
||||
// Member-based filter: an MC matches if ANY of its atomic members has the
|
||||
@@ -141,12 +163,23 @@ async function handleControls(params: URLSearchParams) {
|
||||
const sort = params.get('sort') || 'control_id'
|
||||
const order = params.get('order') === 'desc' ? 'DESC' : 'ASC'
|
||||
|
||||
const { where, args, idx } = buildControlsWhere(params)
|
||||
const hasMapping = await hasMappingTables()
|
||||
const { where, args, idx } = buildControlsWhere(params, hasMapping)
|
||||
|
||||
const sortCol = sort === 'control_id' ? 'mc.master_control_id' :
|
||||
sort === 'created_at' ? 'mc.created_at' :
|
||||
sort === 'source' ? 'mc.canonical_name' : 'mc.master_control_id'
|
||||
|
||||
const mapCols = hasMapping ? `,
|
||||
(SELECT v.verification_method FROM compliance.mc_verification v
|
||||
WHERE v.master_control_uuid = mc.id) as verification_method,
|
||||
(SELECT array_agg(m.use_case ORDER BY m.is_primary DESC, m.use_case)
|
||||
FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id) as use_cases,
|
||||
(SELECT r.source_regulation FROM compliance.mc_regulations r
|
||||
WHERE r.master_control_uuid = mc.id AND r.is_primary LIMIT 1) as primary_regulation`
|
||||
: `, NULL as verification_method, NULL::text[] as use_cases, NULL as primary_regulation`
|
||||
|
||||
args.push(limit, offset)
|
||||
const res = await pool.query(`
|
||||
SELECT mc.master_control_id as control_id,
|
||||
@@ -159,14 +192,7 @@ async function handleControls(params: URLSearchParams) {
|
||||
mc.total_controls,
|
||||
mc.phases_covered,
|
||||
mc.id,
|
||||
mc.created_at,
|
||||
(SELECT v.verification_method FROM compliance.mc_verification v
|
||||
WHERE v.master_control_uuid = mc.id) as verification_method,
|
||||
(SELECT array_agg(m.use_case ORDER BY m.is_primary DESC, m.use_case)
|
||||
FROM compliance.mc_use_case_mappings m
|
||||
WHERE m.master_control_uuid = mc.id) as use_cases,
|
||||
(SELECT r.source_regulation FROM compliance.mc_regulations r
|
||||
WHERE r.master_control_uuid = mc.id AND r.is_primary LIMIT 1) as primary_regulation
|
||||
mc.created_at${mapCols}
|
||||
FROM compliance.master_controls mc
|
||||
${where}
|
||||
ORDER BY ${sortCol} ${order}
|
||||
@@ -203,7 +229,8 @@ async function handleControls(params: URLSearchParams) {
|
||||
}
|
||||
|
||||
async function handleCount(params: URLSearchParams) {
|
||||
const { where, args } = buildControlsWhere(params)
|
||||
const hasMapping = await hasMappingTables()
|
||||
const { where, args } = buildControlsWhere(params, hasMapping)
|
||||
const res = await pool.query(
|
||||
`SELECT count(*) FROM compliance.master_controls mc ${where}`, args
|
||||
)
|
||||
@@ -230,23 +257,26 @@ async function handleMeta(_params: URLSearchParams) {
|
||||
GROUP BY 1 ORDER BY 2 DESC LIMIT 30
|
||||
`)
|
||||
|
||||
// Mapping distribution + coverage + member-derived category facet. Only
|
||||
// category is populated on the deduplicated members (evidence_type /
|
||||
// target_audience are NULL there), so it is the one canonical facet we surface.
|
||||
const [ucRes, vRes, regRes, mappedRes, catRes] = await Promise.all([
|
||||
pool.query(`SELECT use_case, count(DISTINCT master_control_uuid) c
|
||||
FROM compliance.mc_use_case_mappings GROUP BY 1 ORDER BY 2 DESC`),
|
||||
pool.query(`SELECT verification_method, count(*) c
|
||||
FROM compliance.mc_verification GROUP BY 1 ORDER BY 2 DESC`),
|
||||
pool.query(`SELECT source_regulation, count(DISTINCT master_control_uuid) c
|
||||
FROM compliance.mc_regulations GROUP BY 1 ORDER BY 2 DESC LIMIT 200`),
|
||||
pool.query(`SELECT count(DISTINCT master_control_uuid) c
|
||||
FROM compliance.mc_use_case_mappings`),
|
||||
pool.query(`SELECT cc.category v, count(DISTINCT mcm.master_control_uuid) c
|
||||
// category facet is member-based (those tables always exist); the mapping
|
||||
// facets only when the mapping tables are present (seeded DB).
|
||||
const hasMapping = await hasMappingTables()
|
||||
const catRes = await pool.query(`SELECT cc.category v, count(DISTINCT mcm.master_control_uuid) c
|
||||
FROM compliance.master_control_members mcm
|
||||
JOIN compliance.canonical_controls cc ON cc.id = mcm.control_uuid
|
||||
WHERE cc.category IS NOT NULL GROUP BY 1 ORDER BY 2 DESC`),
|
||||
])
|
||||
WHERE cc.category IS NOT NULL GROUP BY 1 ORDER BY 2 DESC`)
|
||||
const emptyRows = { rows: [] as Array<Record<string, string>> }
|
||||
const [ucRes, vRes, regRes, mappedRes] = hasMapping
|
||||
? await Promise.all([
|
||||
pool.query(`SELECT use_case, count(DISTINCT master_control_uuid) c
|
||||
FROM compliance.mc_use_case_mappings GROUP BY 1 ORDER BY 2 DESC`),
|
||||
pool.query(`SELECT verification_method, count(*) c
|
||||
FROM compliance.mc_verification GROUP BY 1 ORDER BY 2 DESC`),
|
||||
pool.query(`SELECT source_regulation, count(DISTINCT master_control_uuid) c
|
||||
FROM compliance.mc_regulations GROUP BY 1 ORDER BY 2 DESC LIMIT 200`),
|
||||
pool.query(`SELECT count(DISTINCT master_control_uuid) c
|
||||
FROM compliance.mc_use_case_mappings`),
|
||||
])
|
||||
: [emptyRows, emptyRows, emptyRows, { rows: [{ c: '0' }] }]
|
||||
const facet = (rows: Array<{ v: string; c: string }>) =>
|
||||
Object.fromEntries(rows.filter(x => x.v).map(x => [x.v, parseInt(x.c)]))
|
||||
|
||||
@@ -311,8 +341,9 @@ async function handleDetail(params: URLSearchParams) {
|
||||
LIMIT 100
|
||||
`, [mc.id])
|
||||
|
||||
// Use-case / verification / regulation mapping (the "regulation→code" lineage)
|
||||
const mapRes = await pool.query(`
|
||||
// Use-case / verification / regulation mapping (only when the tables exist).
|
||||
const mapping: Record<string, any> = (await hasMappingTables())
|
||||
? ((await pool.query(`
|
||||
SELECT
|
||||
(SELECT json_agg(json_build_object('use_case', m.use_case, 'is_primary', m.is_primary)
|
||||
ORDER BY m.is_primary DESC, m.use_case)
|
||||
@@ -323,8 +354,8 @@ async function handleDetail(params: URLSearchParams) {
|
||||
'is_primary', r.is_primary, 'member_count', r.member_count)
|
||||
ORDER BY r.is_primary DESC, r.member_count DESC)
|
||||
FROM compliance.mc_regulations r WHERE r.master_control_uuid = $1) as regulations
|
||||
`, [mc.id])
|
||||
const mapping = mapRes.rows[0] || {}
|
||||
`, [mc.id])).rows[0] || {})
|
||||
: {}
|
||||
const regs = mapping.regulations || []
|
||||
const primaryReg = regs.find((x: { is_primary: boolean }) => x.is_primary) || regs[0]
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Migration 151: iace_projects.parent_project_id (Variantenmanagement)
|
||||
-- Die Spalte wurde urspruenglich ad-hoc auf die DBs gebracht (Commit 8682522,
|
||||
-- 2026-05-09) OHNE Repo-Migration -- dadurch ging sie bei DB-Wechseln/Resets
|
||||
-- verloren. Diese Migration macht sie reproduzierbar. Code erwartet
|
||||
-- *uuid.UUID (nullable). Strikt add-only. [migration-approved]
|
||||
|
||||
SET search_path TO compliance, public;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema='compliance' AND table_name='iace_projects') THEN
|
||||
ALTER TABLE iace_projects
|
||||
ADD COLUMN IF NOT EXISTS parent_project_id UUID;
|
||||
END IF;
|
||||
END $$;
|
||||
+4
-4
@@ -204,10 +204,10 @@ services:
|
||||
expose:
|
||||
- "8095"
|
||||
environment:
|
||||
MINIO_ENDPOINT: nbg1.your-objectstorage.com
|
||||
MINIO_ACCESS_KEY: T18RGFVXXG2ZHQ5404TP
|
||||
MINIO_SECRET_KEY: KOUU4WO6wh07cQjNgh0IZHkeKQrVfBz6hnIGpNss
|
||||
MINIO_SECURE: "true"
|
||||
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-nbg1.your-objectstorage.com}
|
||||
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-T18RGFVXXG2ZHQ5404TP}
|
||||
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-KOUU4WO6wh07cQjNgh0IZHkeKQrVfBz6hnIGpNss}
|
||||
MINIO_SECURE: ${MINIO_SECURE:-true}
|
||||
PIPER_MODEL_PATH: /app/models/de_DE-thorsten-high.onnx
|
||||
depends_on:
|
||||
core-health-check:
|
||||
|
||||
Reference in New Issue
Block a user