feat: DSFA vollständiges DB-Schema + PDF-Ingest + Tests
All checks were successful
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-ai-compliance (push) Successful in 36s
CI / test-python-backend-compliance (push) Successful in 37s
CI / test-python-document-crawler (push) Successful in 23s
CI / test-python-dsms-gateway (push) Successful in 22s

- Migration 030: alle fehlenden Spalten für compliance_dsfas (Sections 0-7)
  flat fields: processing_description, legal_basis, dpo_*, authority_*, ...
  JSONB arrays: risks, mitigations, wp248_criteria_met, ai_trigger_ids, ...
  JSONB objects: section_progress, threshold_analysis, review_schedule, metadata
- dsfa_routes.py: DSFACreate/DSFAUpdate erweitert (60+ neue Optional-Felder)
  _dsfa_to_response: alle neuen Felder mit safe _get() Helper
  PUT-Handler: vollständige JSONB_FIELDS-Liste (22 Felder)
- Tests: 101 (+49) Tests — TestAIUseCaseModules + TestDSFAFullSchema
- ingest-dsfa-bundesland.sh: KNOWN_PDF_URLS (15 direkte URLs), download_pdfs()
  find_pdf_for_state() Helper, PDF-first mit Text-Fallback in ingest_all()

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-05 10:03:09 +01:00
parent ff765b2d71
commit 789c215e5e
4 changed files with 774 additions and 42 deletions

View File

@@ -0,0 +1,73 @@
-- Migration 030: DSFA Vollständiges Schema — alle Felder für Sections 07
-- Fügt alle fehlenden Spalten zur compliance_dsfas Tabelle hinzu.
-- TypeScript-Typen in types.ts erwarten 60+ Felder; bisher waren nur 15 vorhanden.
-- Section 1: Verarbeitungsbeschreibung + Rechtsgrundlage
ALTER TABLE compliance.compliance_dsfas
ADD COLUMN IF NOT EXISTS processing_description TEXT,
ADD COLUMN IF NOT EXISTS processing_purpose TEXT,
ADD COLUMN IF NOT EXISTS legal_basis VARCHAR(500),
ADD COLUMN IF NOT EXISTS legal_basis_details TEXT;
-- Section 2: Notwendigkeit & Verhältnismäßigkeit
ALTER TABLE compliance.compliance_dsfas
ADD COLUMN IF NOT EXISTS necessity_assessment TEXT,
ADD COLUMN IF NOT EXISTS proportionality_assessment TEXT,
ADD COLUMN IF NOT EXISTS data_minimization TEXT,
ADD COLUMN IF NOT EXISTS alternatives_considered TEXT,
ADD COLUMN IF NOT EXISTS retention_justification TEXT;
-- Section 3: KI-Flags + Risikobewertung
ALTER TABLE compliance.compliance_dsfas
ADD COLUMN IF NOT EXISTS involves_ai BOOLEAN DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS overall_risk_level VARCHAR(50),
ADD COLUMN IF NOT EXISTS risk_score INTEGER DEFAULT 0;
-- Section 6: DSB & Aufsichtsbehörde
ALTER TABLE compliance.compliance_dsfas
ADD COLUMN IF NOT EXISTS dpo_consulted BOOLEAN DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS dpo_consulted_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS dpo_name VARCHAR(255),
ADD COLUMN IF NOT EXISTS dpo_opinion TEXT,
ADD COLUMN IF NOT EXISTS dpo_approved BOOLEAN,
ADD COLUMN IF NOT EXISTS authority_consulted BOOLEAN DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS authority_consulted_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS authority_reference VARCHAR(255),
ADD COLUMN IF NOT EXISTS authority_decision TEXT;
-- Versionierung & Metadaten
ALTER TABLE compliance.compliance_dsfas
ADD COLUMN IF NOT EXISTS version INTEGER DEFAULT 1,
ADD COLUMN IF NOT EXISTS previous_version_id UUID,
ADD COLUMN IF NOT EXISTS conclusion TEXT,
ADD COLUMN IF NOT EXISTS federal_state VARCHAR(100),
ADD COLUMN IF NOT EXISTS authority_resource_id VARCHAR(100),
ADD COLUMN IF NOT EXISTS submitted_for_review_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS submitted_by VARCHAR(255);
-- JSONB Arrays
ALTER TABLE compliance.compliance_dsfas
ADD COLUMN IF NOT EXISTS data_subjects JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS affected_rights JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS triggered_rule_codes JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS ai_trigger_ids JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS wp248_criteria_met JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS art35_abs3_triggered JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS tom_references JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS risks JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS mitigations JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS stakeholder_consultations JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS review_triggers JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS review_comments JSONB DEFAULT '[]'::jsonb;
-- JSONB Objekte
ALTER TABLE compliance.compliance_dsfas
ADD COLUMN IF NOT EXISTS threshold_analysis JSONB,
ADD COLUMN IF NOT EXISTS consultation_requirement JSONB,
ADD COLUMN IF NOT EXISTS review_schedule JSONB,
ADD COLUMN IF NOT EXISTS section_progress JSONB DEFAULT '{}'::jsonb,
ADD COLUMN IF NOT EXISTS metadata JSONB DEFAULT '{}'::jsonb;
-- Indizes für häufig gefilterte Spalten
CREATE INDEX IF NOT EXISTS idx_dsfas_federal_state ON compliance.compliance_dsfas(federal_state);
CREATE INDEX IF NOT EXISTS idx_dsfas_involves_ai ON compliance.compliance_dsfas(involves_ai);