From 637fab6fdb4d4b41540d7dcad310dcc0c3043773 Mon Sep 17 00:00:00 2001 From: Benjamin Admin Date: Sat, 14 Mar 2026 21:59:10 +0100 Subject: [PATCH] fix: migration runner strips BEGIN/COMMIT and guards missing tables Root cause: migrations 046-047 used explicit BEGIN/COMMIT which conflicts with psycopg2 implicit transactions, and ALTER TABLE on canonical_controls fails when the table doesn't exist on production. This blocked all subsequent migrations (048-053). Changes: - migration_runner.py: strip BEGIN/COMMIT from SQL before executing - 046: wrap canonical_controls ALTER in DO $$ IF EXISTS block - 047: wrap canonical_controls ALTER in DO $$ IF EXISTS block Co-Authored-By: Claude Opus 4.6 --- backend-compliance/migration_runner.py | 4 +- .../migrations/046_control_generator.sql | 48 +++++++------------ .../047_verification_method_category.sql | 22 +++++---- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/backend-compliance/migration_runner.py b/backend-compliance/migration_runner.py index e275a90..d21bca7 100644 --- a/backend-compliance/migration_runner.py +++ b/backend-compliance/migration_runner.py @@ -84,7 +84,9 @@ def run_migrations(): logger.info("Applying migration: %s", migration_file.name) try: sql = migration_file.read_text(encoding="utf-8") - # Execute the full SQL file as-is (supports BEGIN/COMMIT) + # Strip explicit BEGIN/COMMIT — we manage transactions ourselves + sql = re.sub(r'(?mi)^\s*BEGIN\s*;\s*$', '', sql) + sql = re.sub(r'(?mi)^\s*COMMIT\s*;\s*$', '', sql) cursor.execute(sql) raw_conn.commit() # Record successful application diff --git a/backend-compliance/migrations/046_control_generator.sql b/backend-compliance/migrations/046_control_generator.sql index 08ea4e9..bc8ecc4 100644 --- a/backend-compliance/migrations/046_control_generator.sql +++ b/backend-compliance/migrations/046_control_generator.sql @@ -2,7 +2,7 @@ -- Adds job tracking, chunk tracking, blocked sources, and extends canonical_controls -- for the 3-license-rule system (free_use, citation_required, restricted). -BEGIN; +-- Transaction managed by migration_runner -- ============================================================================= -- 1. Job-Tracking for Generator Runs @@ -69,35 +69,21 @@ CREATE TABLE IF NOT EXISTS canonical_blocked_sources ( -- ============================================================================= -- 4. Extend canonical_controls: release_state + 3-rule columns +-- Safe: only runs if canonical_controls exists -- ============================================================================= --- Expand release_state enum to include generator states -ALTER TABLE canonical_controls DROP CONSTRAINT IF EXISTS canonical_controls_release_state_check; -ALTER TABLE canonical_controls ADD CONSTRAINT canonical_controls_release_state_check - CHECK (release_state IN ('draft', 'review', 'approved', 'deprecated', 'needs_review', 'too_close', 'duplicate')); - --- License rule: 1 = free_use, 2 = citation_required, 3 = restricted -ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS - license_rule INTEGER DEFAULT NULL; - --- Original text from source (Rule 1+2 only; Rule 3 = always NULL) -ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS - source_original_text TEXT DEFAULT NULL; - --- Citation info (Rule 1+2 only; Rule 3 = always NULL) -ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS - source_citation JSONB DEFAULT NULL; - --- Whether source info may be shown to customers -ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS - customer_visible BOOLEAN DEFAULT true; - --- Generation metadata (internal only, never shown to customers) -ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS - generation_metadata JSONB DEFAULT NULL; - --- Index for filtering by license rule and customer visibility -CREATE INDEX IF NOT EXISTS idx_canonical_controls_license_rule ON canonical_controls(license_rule); -CREATE INDEX IF NOT EXISTS idx_canonical_controls_customer_visible ON canonical_controls(customer_visible); - -COMMIT; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'canonical_controls') THEN + ALTER TABLE canonical_controls DROP CONSTRAINT IF EXISTS canonical_controls_release_state_check; + ALTER TABLE canonical_controls ADD CONSTRAINT canonical_controls_release_state_check + CHECK (release_state IN ('draft', 'review', 'approved', 'deprecated', 'needs_review', 'too_close', 'duplicate')); + ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS license_rule INTEGER DEFAULT NULL; + ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS source_original_text TEXT DEFAULT NULL; + ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS source_citation JSONB DEFAULT NULL; + ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS customer_visible BOOLEAN DEFAULT true; + ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS generation_metadata JSONB DEFAULT NULL; + CREATE INDEX IF NOT EXISTS idx_canonical_controls_license_rule ON canonical_controls(license_rule); + CREATE INDEX IF NOT EXISTS idx_canonical_controls_customer_visible ON canonical_controls(customer_visible); + END IF; +END $$; diff --git a/backend-compliance/migrations/047_verification_method_category.sql b/backend-compliance/migrations/047_verification_method_category.sql index 1293322..d04308e 100644 --- a/backend-compliance/migrations/047_verification_method_category.sql +++ b/backend-compliance/migrations/047_verification_method_category.sql @@ -1,16 +1,20 @@ -- Migration 047: Add verification_method and category to canonical_controls -- verification_method: How a control is verified (code_review, document, tool, hybrid) -- category: Thematic grouping for customer-facing filters +-- Safe: only alters canonical_controls if it exists -ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS - verification_method VARCHAR(20) DEFAULT NULL - CHECK (verification_method IN ('code_review', 'document', 'tool', 'hybrid')); - -ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS - category VARCHAR(50) DEFAULT NULL; - -CREATE INDEX IF NOT EXISTS idx_cc_verification ON canonical_controls(verification_method); -CREATE INDEX IF NOT EXISTS idx_cc_category ON canonical_controls(category); +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'canonical_controls') THEN + ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS + verification_method VARCHAR(20) DEFAULT NULL + CHECK (verification_method IN ('code_review', 'document', 'tool', 'hybrid')); + ALTER TABLE canonical_controls ADD COLUMN IF NOT EXISTS + category VARCHAR(50) DEFAULT NULL; + CREATE INDEX IF NOT EXISTS idx_cc_verification ON canonical_controls(verification_method); + CREATE INDEX IF NOT EXISTS idx_cc_category ON canonical_controls(category); + END IF; +END $$; CREATE TABLE IF NOT EXISTS canonical_control_categories ( category_id VARCHAR(50) PRIMARY KEY,