"""
DSFA Corpus Ingestion Pipeline.

Indexes DSFA guidance documents into Qdrant with full source attribution.

Collections:
- bp_dsfa_corpus: All DSFA-related documents (WP248, DSK, Muss-Listen)

Usage:
    python dsfa_corpus_ingestion.py --init-sources    # Register all sources
    python dsfa_corpus_ingestion.py --ingest WP248    # Ingest specific source
    python dsfa_corpus_ingestion.py --ingest-all      # Ingest all sources
    python dsfa_corpus_ingestion.py --status          # Show ingestion status
"""

import os
import re
import hashlib
import uuid
import asyncio
import argparse
from typing import List, Dict, Optional, Any
from dataclasses import dataclass, field, asdict
from datetime import datetime
from enum import Enum

import asyncpg
from qdrant_client import QdrantClient
from qdrant_client.models import (
    VectorParams, Distance, PointStruct, Filter, FieldCondition, MatchValue
)

# Configuration
QDRANT_URL = os.getenv("QDRANT_URL", "http://qdrant:6333")
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://breakpilot:breakpilot@postgres:5432/breakpilot_db")
MINIO_BUCKET = "dsfa-documents"

# DSFA Collection Configuration
DSFA_COLLECTION = "bp_dsfa_corpus"
VECTOR_SIZE = 1024  # BGE-M3


# =============================================================================
# License Registry
# =============================================================================

LICENSE_REGISTRY = {
    "DL-DE-BY-2.0": {
        "name": "Datenlizenz Deutschland – Namensnennung – Version 2.0",
        "url": "https://www.govdata.de/dl-de/by-2-0",
        "attribution_required": True,
        "modification_allowed": True,
        "commercial_use": True,
        "template": "Quelle: {source_name}, Datenlizenz Deutschland – Namensnennung – Version 2.0"
    },
    "DL-DE-ZERO-2.0": {
        "name": "Datenlizenz Deutschland – Zero – Version 2.0",
        "url": "https://www.govdata.de/dl-de/zero-2-0",
        "attribution_required": False,
        "modification_allowed": True,
        "commercial_use": True,
        "template": None
    },
    "CC-BY-4.0": {
        "name": "Creative Commons Attribution 4.0 International",
        "url": "https://creativecommons.org/licenses/by/4.0/",
        "attribution_required": True,
        "modification_allowed": True,
        "commercial_use": True,
        "template": "© {organization} | CC BY 4.0"
    },
    "EDPB-LICENSE": {
        "name": "EDPB Document License",
        "url": "https://edpb.europa.eu/about-edpb/legal-notice_en",
        "attribution_required": True,
        "modification_allowed": True,
        "commercial_use": True,
        "template": "Source: {source_name}, European Data Protection Board"
    },
    "PUBLIC_DOMAIN": {
        "name": "Public Domain",
        "url": None,
        "attribution_required": False,
        "modification_allowed": True,
        "commercial_use": True,
        "template": None
    },
    "PROPRIETARY": {
        "name": "Proprietary (internal use only)",
        "url": None,
        "attribution_required": False,
        "modification_allowed": False,
        "commercial_use": True,
        "template": "© BreakPilot - Internal Use Only"
    },
    "OGL-3.0": {
        "name": "Open Government Licence v3.0",
        "url": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/",
        "attribution_required": True,
        "modification_allowed": True,
        "commercial_use": True,
        "template": "Contains public sector information licensed under the Open Government Licence v3.0. Source: {source_name}"
    }
}


# =============================================================================
# DSFA Sources Registry
# =============================================================================

DSFA_SOURCES = [
    # === Primärquellen (EU/DSGVO) ===
    {
        "source_code": "GDPR_ART35",
        "name": "Art. 35 DSGVO - DSFA",
        "full_name": "Datenschutz-Folgenabschätzung gemäß Artikel 35 DSGVO",
        "organization": "Europäische Union",
        "source_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/deu",
        "eur_lex_celex": "32016R0679",
        "license_code": "CC-BY-4.0",
        "attribution_text": "Quelle: DSGVO Art. 35 (EUR-Lex)",
        "document_type": "regulation",
        "language": "de"
    },
    {
        "source_code": "GDPR_ART36",
        "name": "Art. 36 DSGVO - Behördenkonsultation",
        "full_name": "Vorherige Konsultation gemäß Artikel 36 DSGVO",
        "organization": "Europäische Union",
        "source_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/deu",
        "eur_lex_celex": "32016R0679",
        "license_code": "CC-BY-4.0",
        "attribution_text": "Quelle: DSGVO Art. 36 (EUR-Lex)",
        "document_type": "regulation",
        "language": "de"
    },
    {
        "source_code": "GDPR_RECITALS",
        "name": "Erwägungsgründe 75, 84, 89-91 DSGVO",
        "full_name": "Erwägungsgründe zur Datenschutz-Folgenabschätzung",
        "organization": "Europäische Union",
        "source_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/deu",
        "eur_lex_celex": "32016R0679",
        "license_code": "CC-BY-4.0",
        "attribution_text": "Quelle: DSGVO Erwägungsgründe (EUR-Lex)",
        "document_type": "regulation",
        "language": "de"
    },

    # === WP29/EDPB Leitlinien ===
    {
        "source_code": "WP248",
        "name": "WP248 rev.01 - Leitlinien zur DSFA",
        "full_name": "Leitlinien zur Datenschutz-Folgenabschätzung und Beantwortung der Frage, ob eine Verarbeitung 'wahrscheinlich ein hohes Risiko' birgt",
        "organization": "Artikel-29-Datenschutzgruppe / EDPB",
        "source_url": "https://ec.europa.eu/newsroom/article29/items/611236/en",
        "license_code": "EDPB-LICENSE",
        "attribution_text": "Quelle: WP248 rev.01, Artikel-29-Datenschutzgruppe (2017), bestätigt durch EDPB",
        "document_type": "guideline",
        "language": "de"
    },

    # === DSK Dokumente ===
    {
        "source_code": "DSK_KP5",
        "name": "Kurzpapier Nr. 5 - DSFA nach Art. 35 DS-GVO",
        "full_name": "DSK Kurzpapier Nr. 5: Datenschutz-Folgenabschätzung nach Art. 35 DS-GVO",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutzkonferenz-online.de/media/kp/dsk_kpnr_5.pdf",
        "license_code": "DL-DE-BY-2.0",
        "license_url": "https://www.govdata.de/dl-de/by-2-0",
        "attribution_text": "Quelle: DSK Kurzpapier Nr. 5 (Stand: 2018), Datenlizenz Deutschland – Namensnennung – Version 2.0",
        "document_type": "guideline",
        "language": "de"
    },

    # === Muss-Listen Bund ===
    {
        "source_code": "BFDI_MUSS_PUBLIC",
        "name": "BfDI DSFA-Liste (öffentlicher Bereich)",
        "full_name": "Liste der Verarbeitungsvorgänge nach Art. 35 Abs. 4 DSGVO - Öffentlicher Bereich",
        "organization": "Bundesbeauftragter für den Datenschutz und die Informationsfreiheit",
        "source_url": "https://www.bfdi.bund.de/SharedDocs/Downloads/DE/Muster/Liste_VerarbeitungsvorgaengeArt35.pdf",
        "license_code": "DL-DE-ZERO-2.0",
        "attribution_text": "Quelle: BfDI, Liste gem. Art. 35 Abs. 4 DSGVO (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "BFDI_MUSS_PRIVATE",
        "name": "BfDI DSFA-Liste (nicht-öffentlicher Bereich)",
        "full_name": "Liste der Verarbeitungsvorgänge nach Art. 35 Abs. 4 DSGVO - Nicht-öffentlicher Bereich",
        "organization": "Bundesbeauftragter für den Datenschutz und die Informationsfreiheit",
        "source_url": "https://www.bfdi.bund.de/SharedDocs/Downloads/DE/Muster/Liste_VerarbeitungsvorgaengeArt35.pdf",
        "license_code": "DL-DE-ZERO-2.0",
        "attribution_text": "Quelle: BfDI, Liste gem. Art. 35 Abs. 4 DSGVO (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },

    # === Muss-Listen Länder ===
    # Baden-Württemberg
    {
        "source_code": "BW_MUSS_PUBLIC",
        "name": "LfDI BW DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz BW",
        "source_url": "https://www.baden-wuerttemberg.datenschutz.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Baden-Württemberg, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "BW_MUSS_PRIVATE",
        "name": "LfDI BW DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz BW",
        "source_url": "https://www.baden-wuerttemberg.datenschutz.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Baden-Württemberg, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Bayern
    {
        "source_code": "BY_MUSS_PUBLIC",
        "name": "BayLDA DSFA-Liste (öffentlich)",
        "organization": "Bayerisches Landesamt für Datenschutzaufsicht",
        "source_url": "https://www.lda.bayern.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: BayLDA, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "BY_MUSS_PRIVATE",
        "name": "BayLDA DSFA-Liste (nicht-öffentlich)",
        "organization": "Bayerisches Landesamt für Datenschutzaufsicht",
        "source_url": "https://www.lda.bayern.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: BayLDA, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Berlin
    {
        "source_code": "BE_MUSS_PUBLIC",
        "name": "BlnBDI DSFA-Liste (öffentlich)",
        "organization": "Berliner Beauftragte für Datenschutz",
        "source_url": "https://www.datenschutz-berlin.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: BlnBDI, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "BE_MUSS_PRIVATE",
        "name": "BlnBDI DSFA-Liste (nicht-öffentlich)",
        "organization": "Berliner Beauftragte für Datenschutz",
        "source_url": "https://www.datenschutz-berlin.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: BlnBDI, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Brandenburg
    {
        "source_code": "BB_MUSS_PUBLIC",
        "name": "LDA BB DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz Brandenburg",
        "source_url": "https://www.lda.brandenburg.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LDA Brandenburg, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "BB_MUSS_PRIVATE",
        "name": "LDA BB DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz Brandenburg",
        "source_url": "https://www.lda.brandenburg.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LDA Brandenburg, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Bremen
    {
        "source_code": "HB_MUSS_PUBLIC",
        "name": "LfDI HB DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz Bremen",
        "source_url": "https://www.datenschutz.bremen.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Bremen, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "HB_MUSS_PRIVATE",
        "name": "LfDI HB DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz Bremen",
        "source_url": "https://www.datenschutz.bremen.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Bremen, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Hamburg
    {
        "source_code": "HH_MUSS_PUBLIC",
        "name": "HmbBfDI DSFA-Liste (öffentlich)",
        "organization": "Hamburgische Beauftragte für Datenschutz",
        "source_url": "https://datenschutz-hamburg.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: HmbBfDI, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "HH_MUSS_PRIVATE",
        "name": "HmbBfDI DSFA-Liste (nicht-öffentlich)",
        "organization": "Hamburgische Beauftragte für Datenschutz",
        "source_url": "https://datenschutz-hamburg.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: HmbBfDI, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Hessen
    {
        "source_code": "HE_MUSS_PUBLIC",
        "name": "HBDI DSFA-Liste (öffentlich)",
        "organization": "Hessischer Beauftragter für Datenschutz",
        "source_url": "https://datenschutz.hessen.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: HBDI, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "HE_MUSS_PRIVATE",
        "name": "HBDI DSFA-Liste (nicht-öffentlich)",
        "organization": "Hessischer Beauftragter für Datenschutz",
        "source_url": "https://datenschutz.hessen.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: HBDI, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Mecklenburg-Vorpommern
    {
        "source_code": "MV_MUSS_PUBLIC",
        "name": "LfDI MV DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz MV",
        "source_url": "https://www.datenschutz-mv.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI MV, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "MV_MUSS_PRIVATE",
        "name": "LfDI MV DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz MV",
        "source_url": "https://www.datenschutz-mv.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI MV, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Niedersachsen
    {
        "source_code": "NI_MUSS_PUBLIC",
        "name": "LfD NI DSFA-Liste (öffentlich)",
        "organization": "Die Landesbeauftragte für den Datenschutz Niedersachsen",
        "source_url": "https://www.lfd.niedersachsen.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfD Niedersachsen, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "NI_MUSS_PRIVATE",
        "name": "LfD NI DSFA-Liste (nicht-öffentlich)",
        "organization": "Die Landesbeauftragte für den Datenschutz Niedersachsen",
        "source_url": "https://www.lfd.niedersachsen.de/download/131098",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfD Niedersachsen, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Nordrhein-Westfalen
    {
        "source_code": "NW_MUSS_PUBLIC",
        "name": "LDI NRW DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz NRW",
        "source_url": "https://www.ldi.nrw.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LDI NRW, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "NW_MUSS_PRIVATE",
        "name": "LDI NRW DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz NRW",
        "source_url": "https://www.ldi.nrw.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LDI NRW, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Rheinland-Pfalz
    {
        "source_code": "RP_MUSS_PUBLIC",
        "name": "LfDI RP DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz Rheinland-Pfalz",
        "source_url": "https://www.datenschutz.rlp.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Rheinland-Pfalz, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "RP_MUSS_PRIVATE",
        "name": "LfDI RP DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz Rheinland-Pfalz",
        "source_url": "https://www.datenschutz.rlp.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Rheinland-Pfalz, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Saarland
    {
        "source_code": "SL_MUSS_PUBLIC",
        "name": "LfDI SL DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz Saarland",
        "source_url": "https://www.datenschutz.saarland.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Saarland, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "SL_MUSS_PRIVATE",
        "name": "LfDI SL DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragte für Datenschutz Saarland",
        "source_url": "https://www.datenschutz.saarland.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfDI Saarland, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Sachsen
    {
        "source_code": "SN_MUSS_PUBLIC",
        "name": "SDB DSFA-Liste (öffentlich)",
        "organization": "Sächsischer Datenschutzbeauftragter",
        "source_url": "https://www.saechsdsb.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: SDB Sachsen, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "SN_MUSS_PRIVATE",
        "name": "SDB DSFA-Liste (nicht-öffentlich)",
        "organization": "Sächsischer Datenschutzbeauftragter",
        "source_url": "https://www.saechsdsb.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: SDB Sachsen, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Sachsen-Anhalt
    {
        "source_code": "ST_MUSS_PUBLIC",
        "name": "LfD ST DSFA-Liste (öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz Sachsen-Anhalt",
        "source_url": "https://datenschutz.sachsen-anhalt.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfD Sachsen-Anhalt, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "ST_MUSS_PRIVATE",
        "name": "LfD ST DSFA-Liste (nicht-öffentlich)",
        "organization": "Landesbeauftragter für Datenschutz Sachsen-Anhalt",
        "source_url": "https://datenschutz.sachsen-anhalt.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: LfD Sachsen-Anhalt, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Schleswig-Holstein
    {
        "source_code": "SH_MUSS_PUBLIC",
        "name": "ULD DSFA-Liste (öffentlich)",
        "organization": "Unabhängiges Landeszentrum für Datenschutz SH",
        "source_url": "https://www.datenschutzzentrum.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: ULD Schleswig-Holstein, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "SH_MUSS_PRIVATE",
        "name": "ULD DSFA-Liste (nicht-öffentlich)",
        "organization": "Unabhängiges Landeszentrum für Datenschutz SH",
        "source_url": "https://www.datenschutzzentrum.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: ULD Schleswig-Holstein, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    # Thüringen
    {
        "source_code": "TH_MUSS_PUBLIC",
        "name": "TLfDI DSFA-Liste (öffentlich)",
        "organization": "Thüringer Landesbeauftragter für Datenschutz",
        "source_url": "https://www.tlfdi.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: TLfDI Thüringen, DSFA-Muss-Liste (öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },
    {
        "source_code": "TH_MUSS_PRIVATE",
        "name": "TLfDI DSFA-Liste (nicht-öffentlich)",
        "organization": "Thüringer Landesbeauftragter für Datenschutz",
        "source_url": "https://www.tlfdi.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: TLfDI Thüringen, DSFA-Muss-Liste (nicht-öffentlicher Bereich)",
        "document_type": "checklist",
        "language": "de"
    },

    # === Sonstige ===
    {
        "source_code": "AI_ACT_DSFA",
        "name": "AI Act Bezüge zu DSFA",
        "full_name": "AI Act Artikel mit Bezug zur Datenschutz-Folgenabschätzung",
        "organization": "Europäische Union",
        "source_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj",
        "license_code": "CC-BY-4.0",
        "attribution_text": "Quelle: AI Act (EU) 2024/1689, EUR-Lex",
        "document_type": "regulation",
        "language": "de"
    },
    {
        "source_code": "DSK_OH_KI",
        "name": "DSK Orientierungshilfe KI",
        "full_name": "DSK Orientierungshilfe KI und Datenschutz",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutzkonferenz-online.de",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: DSK Orientierungshilfe KI und Datenschutz",
        "document_type": "guideline",
        "language": "de"
    },
    {
        "source_code": "EDSA_GUIDELINES",
        "name": "EDPB Guidelines on DPIA",
        "full_name": "European Data Protection Board Guidelines on DPIA",
        "organization": "European Data Protection Board",
        "source_url": "https://edpb.europa.eu",
        "license_code": "EDPB-LICENSE",
        "attribution_text": "Source: EDPB Guidelines on Data Protection Impact Assessment",
        "document_type": "guideline",
        "language": "en"
    },

    # === DSK Weitere Kurzpapiere ===
    {
        "source_code": "DSK_KP18",
        "name": "Kurzpapier Nr. 18 - Risiko für die Rechte und Freiheiten",
        "full_name": "DSK Kurzpapier Nr. 18: Risiko für die Rechte und Freiheiten natürlicher Personen",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutzkonferenz-online.de/media/kp/dsk_kpnr_18.pdf",
        "license_code": "DL-DE-BY-2.0",
        "license_url": "https://www.govdata.de/dl-de/by-2-0",
        "attribution_text": "Quelle: DSK Kurzpapier Nr. 18 (Risiko), Datenlizenz Deutschland – Namensnennung – Version 2.0",
        "document_type": "guideline",
        "language": "de"
    },

    # === Standard-Datenschutzmodell ===
    {
        "source_code": "SDM_V2",
        "name": "Standard-Datenschutzmodell V2.0",
        "full_name": "SDM-Methode der Datenschutzaufsichtsbehörden V2.0",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutzkonferenz-online.de/media/ah/20191106_SDM-Methode_V2.0.pdf",
        "license_code": "DL-DE-BY-2.0",
        "license_url": "https://www.govdata.de/dl-de/by-2-0",
        "attribution_text": "Quelle: SDM V2.0, Datenschutzkonferenz (DSK), Datenlizenz Deutschland – Namensnennung – Version 2.0",
        "document_type": "methodology",
        "language": "de"
    },

    # === Internes Dokument ===
    {
        "source_code": "BREAKPILOT_DSFA_GUIDE",
        "name": "Datenschutz-Folgenabschätzung in Deutschland",
        "full_name": "BreakPilot DSFA-Leitfaden (intern)",
        "organization": "BreakPilot",
        "source_url": None,
        "license_code": "PROPRIETARY",
        "attribution_text": "Quelle: BreakPilot DSFA-Leitfaden (intern)",
        "document_type": "guideline",
        "language": "de"
    },
    {
        "source_code": "BREAKPILOT_BASELINE",
        "name": "Baseline-DSFA Katalog",
        "full_name": "BreakPilot Baseline-DSFA Katalog (proprietär)",
        "organization": "BreakPilot",
        "source_url": None,
        "license_code": "PROPRIETARY",
        "attribution_text": "Quelle: BreakPilot Baseline-DSFA Katalog (intern)",
        "document_type": "catalog",
        "language": "de"
    },
    {
        "source_code": "BREAKPILOT_DSFA_DE",
        "name": "DSFA in Deutschland Dokument",
        "full_name": "BreakPilot DSFA in Deutschland (proprietär)",
        "organization": "BreakPilot",
        "source_url": None,
        "license_code": "PROPRIETARY",
        "attribution_text": "Quelle: BreakPilot DSFA in Deutschland (intern)",
        "document_type": "guideline",
        "language": "de"
    },

    # === VVT-Quellen (Verarbeitungsverzeichnis Art. 30 DSGVO) ===
    {
        "source_code": "DSK_KP1",
        "name": "Kurzpapier Nr. 1 - Verarbeitungsverzeichnis",
        "full_name": "DSK Kurzpapier Nr. 1: Verzeichnis von Verarbeitungstaetigkeiten nach Art. 30 DS-GVO",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutzkonferenz-online.de/media/kp/dsk_kpnr_1.pdf",
        "license_code": "DL-DE-BY-2.0",
        "license_url": "https://www.govdata.de/dl-de/by-2-0",
        "attribution_text": "Quelle: DSK Kurzpapier Nr. 1 (Stand: 2018), Datenlizenz Deutschland – Namensnennung – Version 2.0",
        "document_type": "guideline",
        "language": "de"
    },
    {
        "source_code": "ICO_ROPA",
        "name": "ICO Records of Processing Activities",
        "full_name": "ICO Guidance on Documentation and Records of Processing Activities (RoPA)",
        "organization": "Information Commissioner's Office (ICO)",
        "source_url": "https://ico.org.uk/for-organisations/uk-gdpr-guidance-and-resources/accountability-and-governance/documentation-record-of-processing-activities/",
        "license_code": "OGL-3.0",
        "license_url": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/",
        "attribution_text": "Contains public sector information licensed under the Open Government Licence v3.0. Source: ICO RoPA Guidance",
        "document_type": "guideline",
        "language": "en"
    },
    {
        "source_code": "BREAKPILOT_VVT_SPEC",
        "name": "VVT Generator Spezifikation",
        "full_name": "BreakPilot VVT Generator Spezifikation (proprietaer)",
        "organization": "BreakPilot",
        "source_url": None,
        "license_code": "PROPRIETARY",
        "attribution_text": "Quelle: BreakPilot VVT Generator Spezifikation (intern)",
        "document_type": "specification",
        "language": "de"
    },

    # === SDM Bausteine V3.0 (TOM Gewaehrleistungsziele) ===
    {
        "source_code": "SDM_BAUSTEINE",
        "name": "SDM Bausteine V3.0",
        "full_name": "Standard-Datenschutzmodell Bausteine Version 3.0",
        "organization": "Konferenz der unabhaengigen Datenschutzaufsichtsbehoerden",
        "source_url": "https://www.datenschutz-mv.de/datenschutz/sdm/",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: SDM Bausteine V3.0, Konferenz der unabhaengigen Datenschutzaufsichtsbehoerden des Bundes und der Laender, Lizenz: dl-de/by-2-0",
        "document_type": "standard",
        "language": "de"
    },

    # === DSK Kurzpapier Nr. 7 (Loeschung) ===
    {
        "source_code": "DSK_KP7",
        "name": "DSK Kurzpapier Nr. 7 - Loeschung",
        "full_name": "Kurzpapier Nr. 7: Marktortprinzip und Loeschung personenbezogener Daten",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutzkonferenz-online.de/kurzpapiere.html",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: DSK Kurzpapier Nr. 7, Datenschutzkonferenz, Lizenz: dl-de/by-2-0",
        "document_type": "guidance",
        "language": "de"
    },

    # === BreakPilot Loeschfristen + TOM Spec (intern) ===
    {
        "source_code": "BREAKPILOT_LF_TOM_SPEC",
        "name": "Loeschfristen & TOM Generator Spezifikation",
        "full_name": "BreakPilot Loeschfristen und TOM Generator Spezifikation (proprietaer)",
        "organization": "BreakPilot",
        "source_url": None,
        "license_code": "PROPRIETARY",
        "attribution_text": "Quelle: BreakPilot Loeschfristen & TOM Generator Spezifikation (intern)",
        "document_type": "specification",
        "language": "de"
    },

    # === Compliance Advisor Agent - Zusaetzliche Quellen ===
    {
        "source_code": "DSGVO_VOLLTEXT",
        "name": "DSGVO Volltext",
        "full_name": "Verordnung (EU) 2016/679 - Datenschutz-Grundverordnung (Volltext mit Erwaegsgruenden)",
        "organization": "Europaeische Union",
        "source_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj/deu",
        "license_code": "CC-BY-4.0",
        "attribution_text": "Quelle: DSGVO Volltext, Europaeische Union, CC BY 4.0",
        "document_type": "legislation",
        "language": "de"
    },
    {
        "source_code": "BDSG_VOLLTEXT",
        "name": "BDSG Volltext",
        "full_name": "Bundesdatenschutzgesetz (BDSG) - Volltext",
        "organization": "Bundesrepublik Deutschland",
        "source_url": "https://www.gesetze-im-internet.de/bdsg_2018/",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Quelle: BDSG, Bundesrepublik Deutschland",
        "document_type": "legislation",
        "language": "de"
    },
    {
        "source_code": "AI_ACT_SUMMARY",
        "name": "AI Act Zusammenfassung",
        "full_name": "EU KI-Verordnung (AI Act) - Zusammenfassung und Kernpunkte",
        "organization": "Europaeische Union",
        "source_url": "https://eur-lex.europa.eu/legal-content/DE/TXT/?uri=CELEX:32024R1689",
        "license_code": "CC-BY-4.0",
        "attribution_text": "Quelle: AI Act, Europaeische Union, CC BY 4.0",
        "document_type": "legislation",
        "language": "de"
    },
    {
        "source_code": "DSK_KURZPAPIERE_ALLE",
        "name": "DSK Kurzpapiere (alle 20)",
        "full_name": "Datenschutzkonferenz - Alle 20 Kurzpapiere zur DSGVO",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutzkonferenz-online.de/kurzpapiere.html",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: DSK Kurzpapiere, Datenschutzkonferenz, Lizenz: dl-de/by-2-0",
        "document_type": "guidance",
        "language": "de"
    },
    {
        "source_code": "SDM_V3",
        "name": "Standard-Datenschutzmodell V3.0",
        "full_name": "SDM - Standard-Datenschutzmodell Version 3.0",
        "organization": "Datenschutzkonferenz (DSK)",
        "source_url": "https://www.datenschutz-mv.de/datenschutz/sdm/",
        "license_code": "DL-DE-BY-2.0",
        "attribution_text": "Quelle: SDM V3.0, Datenschutzkonferenz, Lizenz: dl-de/by-2-0",
        "document_type": "standard",
        "language": "de"
    },

    # === EDPB Ergaenzende Leitlinien ===
    # MIGRATED to bp_legal_corpus via migrate_rag_chunks.py (2026-02-10)
    {
        "source_code": "EDPB_GUIDELINES_2_2019",
        "name": "EDPB Leitlinien 2/2019 zu Art. 6(1)(b)",
        "full_name": "EDPB Leitlinien 2/2019 zur Verarbeitung personenbezogener Daten auf Grundlage von Art. 6 Abs. 1 lit. b DSGVO",
        "organization": "European Data Protection Board",
        "source_url": "https://edpb.europa.eu/our-work-tools/our-documents/guidelines/guidelines-22019-processing-personal-data-under-article-61b_en",
        "license_code": "EDPB-LICENSE",
        "attribution_text": "Source: EDPB Guidelines 2/2019, European Data Protection Board",
        "document_type": "guideline",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "EDPB_GUIDELINES_3_2019",
        "name": "EDPB Leitlinien 3/2019 Videoueberwachung",
        "full_name": "EDPB Leitlinien 3/2019 zur Verarbeitung personenbezogener Daten durch Videoueberwachung",
        "organization": "European Data Protection Board",
        "source_url": "https://edpb.europa.eu/our-work-tools/our-documents/guidelines/guidelines-32019-processing-personal-data-through-video_en",
        "license_code": "EDPB-LICENSE",
        "attribution_text": "Source: EDPB Guidelines 3/2019, European Data Protection Board",
        "document_type": "guideline",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "EDPB_GUIDELINES_5_2020",
        "name": "EDPB Leitlinien 5/2020 Einwilligung",
        "full_name": "EDPB Leitlinien 5/2020 zur Einwilligung gemaess Verordnung 2016/679",
        "organization": "European Data Protection Board",
        "source_url": "https://edpb.europa.eu/our-work-tools/our-documents/guidelines/guidelines-052020-consent-under-regulation-2016679_en",
        "license_code": "EDPB-LICENSE",
        "attribution_text": "Source: EDPB Guidelines 5/2020, European Data Protection Board",
        "document_type": "guideline",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "EDPB_GUIDELINES_7_2020",
        "name": "EDPB Leitlinien 7/2020 Controller/Processor",
        "full_name": "EDPB Leitlinien 7/2020 zu den Begriffen Verantwortlicher und Auftragsverarbeiter",
        "organization": "European Data Protection Board",
        "source_url": "https://edpb.europa.eu/our-work-tools/our-documents/guidelines/guidelines-072020-concepts-controller-and-processor-gdpr_en",
        "license_code": "EDPB-LICENSE",
        "attribution_text": "Source: EDPB Guidelines 7/2020, European Data Protection Board",
        "document_type": "guideline",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "EDPB_GUIDELINES_1_2022",
        "name": "EDPB Leitlinien 1/2022 Bussgelder",
        "full_name": "EDPB Leitlinien 04/2022 zur Berechnung von Bussgeldern nach der DSGVO",
        "organization": "European Data Protection Board",
        "source_url": "https://edpb.europa.eu/our-work-tools/our-documents/guidelines/guidelines-042022-calculation-administrative-fines-under-gdpr_en",
        "license_code": "EDPB-LICENSE",
        "attribution_text": "Source: EDPB Guidelines 04/2022, European Data Protection Board",
        "document_type": "guideline",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "SCC_FULL_TEXT",
        "name": "Standard Contractual Clauses Volltext",
        "full_name": "Standardvertragsklauseln fuer die Uebermittlung personenbezogener Daten an Drittlaender (2021/914/EU)",
        "organization": "Europaeische Kommission",
        "source_url": "https://eur-lex.europa.eu/eli/dec_impl/2021/914/oj",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Quelle: SCC Volltext, Europaeische Kommission (EUR-Lex)",
        "document_type": "regulation",
        "language": "de",
        "migrated_to": "bp_legal_corpus"
    },

    # === Nationale Datenschutzgesetze (DSGVO-Umsetzungen) ===
    # MIGRATED to bp_legal_corpus via migrate_rag_chunks.py (2026-02-10)
    # These sources are kept here for reference but will be skipped during ingestion.
    # Ingestion should target bp_legal_corpus for these source codes.
    {
        "source_code": "BDSG_FULL",
        "name": "BDSG Volltext (Deutschland)",
        "full_name": "Bundesdatenschutzgesetz (BDSG) - Volltext inkl. aller Teile",
        "organization": "Bundesrepublik Deutschland",
        "source_url": "https://www.gesetze-im-internet.de/bdsg_2018/",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Quelle: BDSG, Bundesrepublik Deutschland (gesetze-im-internet.de)",
        "document_type": "legislation",
        "language": "de",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "AT_DSG",
        "name": "DSG Oesterreich",
        "full_name": "Bundesgesetz zum Schutz natuerlicher Personen bei der Verarbeitung personenbezogener Daten (Datenschutzgesetz - DSG)",
        "organization": "Republik Oesterreich",
        "source_url": "https://www.ris.bka.gv.at/GeltendeFassung.wxe?Abfrage=Bundesnormen&Gesetzesnummer=10001597",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Quelle: DSG, Republik Oesterreich (RIS)",
        "document_type": "legislation",
        "language": "de",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "CH_DSG",
        "name": "DSG Schweiz (revDSG 2023)",
        "full_name": "Bundesgesetz ueber den Datenschutz (Datenschutzgesetz, DSG) - revidierte Fassung 2023",
        "organization": "Schweizerische Eidgenossenschaft",
        "source_url": "https://www.fedlex.admin.ch/eli/cc/2022/491/de",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Quelle: DSG, Schweizerische Eidgenossenschaft (Fedlex)",
        "document_type": "legislation",
        "language": "de",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "LI_DSG",
        "name": "DSG Liechtenstein",
        "full_name": "Datenschutzgesetz (DSG) Liechtenstein",
        "organization": "Fuerstentum Liechtenstein",
        "source_url": "https://www.gesetze.li/konso/2018.272",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Quelle: DSG, Fuerstentum Liechtenstein (gesetze.li)",
        "document_type": "legislation",
        "language": "de",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "FR_CNIL_GUIDE",
        "name": "CNIL Guide RGPD",
        "full_name": "Guide pratique RGPD - Commission Nationale de l'Informatique et des Libertes",
        "organization": "CNIL (France)",
        "source_url": "https://www.cnil.fr/fr/rgpd-de-quoi-parle-t-on",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Source: CNIL Guide RGPD, Commission Nationale de l'Informatique et des Libertes",
        "document_type": "guideline",
        "language": "fr",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "ES_LOPDGDD",
        "name": "LOPDGDD Spanien",
        "full_name": "Ley Organica de Proteccion de Datos Personales y garantia de los derechos digitales",
        "organization": "Reino de Espana",
        "source_url": "https://www.boe.es/buscar/act.php?id=BOE-A-2018-16673",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Fuente: LOPDGDD, Reino de Espana (BOE)",
        "document_type": "legislation",
        "language": "es",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "IT_CODICE_PRIVACY",
        "name": "Codice Privacy Italien",
        "full_name": "Codice in materia di protezione dei dati personali (D.Lgs. 196/2003, aggiornato D.Lgs. 101/2018)",
        "organization": "Repubblica Italiana",
        "source_url": "https://www.garanteprivacy.it/home/docweb/-/docweb-display/docweb/9042678",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Fonte: Codice Privacy, Garante per la protezione dei dati personali",
        "document_type": "legislation",
        "language": "it",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "NL_UAVG",
        "name": "UAVG Niederlande",
        "full_name": "Uitvoeringswet Algemene verordening gegevensbescherming (UAVG)",
        "organization": "Koninkrijk der Nederlanden",
        "source_url": "https://wetten.overheid.nl/BWBR0040940/",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Bron: UAVG, Koninkrijk der Nederlanden (wetten.overheid.nl)",
        "document_type": "legislation",
        "language": "nl",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "BE_DPA_LAW",
        "name": "Datenschutzgesetz Belgien",
        "full_name": "Loi relative a la protection des personnes physiques a l'egard des traitements de donnees a caractere personnel",
        "organization": "Royaume de Belgique",
        "source_url": "https://www.ejustice.just.fgov.be/cgi_loi/change_lg.pl?language=fr&la=F&cn=2018073046",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Source: Loi Protection des Donnees, Royaume de Belgique (eJustice)",
        "document_type": "legislation",
        "language": "fr",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "LU_DPA_LAW",
        "name": "Datenschutzgesetz Luxemburg",
        "full_name": "Loi du 1er aout 2018 portant organisation de la Commission nationale pour la protection des donnees",
        "organization": "Grand-Duche de Luxembourg",
        "source_url": "https://legilux.public.lu/eli/etat/leg/loi/2018/08/01/a686/jo",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Source: Loi Protection des Donnees, Grand-Duche de Luxembourg (Legilux)",
        "document_type": "legislation",
        "language": "fr",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "IE_DPA_2018",
        "name": "Data Protection Act 2018 Ireland",
        "full_name": "Data Protection Act 2018 (Act No. 7 of 2018) - Ireland",
        "organization": "Government of Ireland",
        "source_url": "https://www.irishstatutebook.ie/eli/2018/act/7/enacted/en/html",
        "license_code": "OGL-3.0",
        "attribution_text": "Contains public sector information licensed under the Open Government Licence v3.0. Source: Data Protection Act 2018, Ireland",
        "document_type": "legislation",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "UK_DPA_2018",
        "name": "Data Protection Act 2018 UK",
        "full_name": "Data Protection Act 2018 (c. 12) - United Kingdom",
        "organization": "Government of the United Kingdom",
        "source_url": "https://www.legislation.gov.uk/ukpga/2018/12/contents/enacted",
        "license_code": "OGL-3.0",
        "attribution_text": "Contains public sector information licensed under the Open Government Licence v3.0. Source: Data Protection Act 2018, UK",
        "document_type": "legislation",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "UK_GDPR",
        "name": "UK GDPR (retained EU law)",
        "full_name": "United Kingdom General Data Protection Regulation (UK GDPR) - retained EU law",
        "organization": "Government of the United Kingdom",
        "source_url": "https://www.legislation.gov.uk/eur/2016/679/contents",
        "license_code": "OGL-3.0",
        "attribution_text": "Contains public sector information licensed under the Open Government Licence v3.0. Source: UK GDPR, legislation.gov.uk",
        "document_type": "legislation",
        "language": "en",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "NO_PERSONOPPLYSNINGSLOVEN",
        "name": "Personopplysningsloven Norwegen",
        "full_name": "Lov om behandling av personopplysninger (personopplysningsloven)",
        "organization": "Kongeriket Norge",
        "source_url": "https://lovdata.no/dokument/NL/lov/2018-06-15-38",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Kilde: Personopplysningsloven, Kongeriket Norge (Lovdata)",
        "document_type": "legislation",
        "language": "no",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "SE_DATASKYDDSLAG",
        "name": "Dataskyddslag Schweden",
        "full_name": "Lag (2018:218) med kompletterande bestammelser till EU:s dataskyddsforordning",
        "organization": "Konungariket Sverige",
        "source_url": "https://www.riksdagen.se/sv/dokument-och-lagar/dokument/svensk-forfattningssamling/lag-2018218-med-kompletterande-bestammelser-till_sfs-2018-218/",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Kalla: Dataskyddslag (2018:218), Konungariket Sverige (Riksdagen)",
        "document_type": "legislation",
        "language": "sv",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "DK_DATABESKYTTELSESLOVEN",
        "name": "Databeskyttelsesloven Daenemark",
        "full_name": "Lov om supplerende bestemmelser til forordning om beskyttelse af fysiske personer i forbindelse med behandling af personoplysninger",
        "organization": "Kongeriget Danmark",
        "source_url": "https://www.retsinformation.dk/eli/lta/2018/502",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Kilde: Databeskyttelsesloven, Kongeriget Danmark (Retsinformation)",
        "document_type": "legislation",
        "language": "da",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "FI_TIETOSUOJALAKI",
        "name": "Tietosuojalaki Finnland",
        "full_name": "Tietosuojalaki (1050/2018) - Datenschutzgesetz Finnland",
        "organization": "Suomen tasavalta",
        "source_url": "https://www.finlex.fi/fi/laki/ajantasa/2018/20181050",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Lahde: Tietosuojalaki, Suomen tasavalta (Finlex)",
        "document_type": "legislation",
        "language": "fi",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "PL_UODO",
        "name": "UODO Polen",
        "full_name": "Ustawa o ochronie danych osobowych - Datenschutzgesetz Polen",
        "organization": "Rzeczpospolita Polska",
        "source_url": "https://isap.sejm.gov.pl/isap.nsf/DocDetails.xsp?id=WDU20180001000",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Zrodlo: Ustawa o ochronie danych osobowych, Rzeczpospolita Polska (ISAP)",
        "document_type": "legislation",
        "language": "pl",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "CZ_ZOU",
        "name": "Zakon o ochrane osobnich udaju Tschechien",
        "full_name": "Zakon c. 110/2019 Sb. o zpracovani osobnich udaju",
        "organization": "Ceska republika",
        "source_url": "https://www.zakonyprolidi.cz/cs/2019-110",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Zdroj: Zakon o ochrane osobnich udaju, Ceska republika (zakonyprolidi.cz)",
        "document_type": "legislation",
        "language": "cs",
        "migrated_to": "bp_legal_corpus"
    },
    {
        "source_code": "HU_INFOTV",
        "name": "Informacios torvenye Ungarn",
        "full_name": "2011. evi CXII. torveny az informacios onrendelkezesi jogrol es az informacioszabadsagrol (Infotv.)",
        "organization": "Magyarorszag",
        "source_url": "https://njt.hu/jogszabaly/2011-112-00-00",
        "license_code": "PUBLIC_DOMAIN",
        "attribution_text": "Forras: Infotv., Magyarorszag (njt.hu)",
        "document_type": "legislation",
        "language": "hu",
        "migrated_to": "bp_legal_corpus"
    },
]


# =============================================================================
# Chunking Configuration
# =============================================================================

DSFA_CHUNK_CONFIG = {
    # WP248 - Kriterien einzeln chunken
    "WP248": {
        "strategy": "section_based",
        "section_markers": [r"K1[:\s]", r"K2[:\s]", r"K3[:\s]", r"K4[:\s]", r"K5[:\s]",
                          r"K6[:\s]", r"K7[:\s]", r"K8[:\s]", r"K9[:\s]"],
        "max_chunk_size": 1500,
        "overlap": 200
    },
    # DSK Kurzpapier - Prozessschritte einzeln
    "DSK_KP5": {
        "strategy": "section_based",
        "section_markers": [r"Schritt\s*1", r"Schritt\s*2", r"Schritt\s*3",
                          r"Schritt\s*4", r"Schritt\s*5", r"Schritt\s*6"],
        "max_chunk_size": 1200,
        "overlap": 150
    },
    # SDM V2.0 - Gewährleistungsziele einzeln
    'SDM_V2': {
        'strategy': 'section_based',
        'section_markers': [
            r'Gewährleistungsziel\s+\d',
            r'\d+\.\d+\s+',
        ],
        'max_chunk_size': 1200,
        'overlap': 150,
        'categories': ['sdm_goal', 'methodology', 'implementation']
    },
    # Muss-Listen - Jeder Eintrag = 1 Chunk
    "MUSS_LISTEN": {
        "strategy": "list_item",
        "list_markers": [r"^•", r"^-", r"^\d+\."],
        "max_chunk_size": 800,
        "overlap": 0
    },
    # DSK Kurzpapier Nr. 1 - VVT Schritte einzeln
    "DSK_KP1": {
        "strategy": "section_based",
        "section_markers": [r"Schritt\s+\d", r"\d+\.\s+"],
        "max_chunk_size": 1000,
        "overlap": 150,
        "categories": ["vvt_guidance", "art30_requirements", "controller_duties"]
    },
    # ICO RoPA - Sections einzeln
    "ICO_ROPA": {
        "strategy": "section_based",
        "section_markers": [r"What\s+should", r"How\s+to", r"Controller", r"Processor"],
        "max_chunk_size": 1000,
        "overlap": 150,
        "categories": ["vvt_guidance", "art30_requirements", "ropa_templates"]
    },
    # SDM Bausteine - Gewaehrleistungsziele einzeln
    "SDM_BAUSTEINE": {
        "strategy": "section_based",
        "section_markers": [
            r"Baustein\s+\d",
            r"Gewaehrleistungsziel",
            r"\d+\.\d+\s+",
        ],
        "max_chunk_size": 1200,
        "overlap": 150,
        "categories": ["sdm_goal", "tom_measure", "implementation"]
    },
    # DSK Kurzpapier Nr. 7 - Loeschung
    "DSK_KP7": {
        "strategy": "section_based",
        "section_markers": [r"Schritt\s+\d", r"\d+\.\s+", r"Loeschkonzept"],
        "max_chunk_size": 1000,
        "overlap": 150,
        "categories": ["loeschung", "art17_requirements", "retention_guidance"]
    },
    # Fallback
    "DEFAULT": {
        "strategy": "recursive",
        "max_chunk_size": 1000,
        "overlap": 200
    }
}


# =============================================================================
# Data Classes
# =============================================================================

@dataclass
class DSFAChunkPayload:
    """Payload schema for Qdrant points."""
    chunk_id: str
    document_id: str
    source_id: str
    content: str
    section_title: Optional[str] = None
    source_code: str = ""
    source_name: str = ""
    attribution_text: str = ""
    license_code: str = ""
    attribution_required: bool = True
    document_type: str = ""
    category: str = ""
    language: str = "de"
    page_number: Optional[int] = None


@dataclass
class DSFASearchResult:
    """Search result with attribution."""
    chunk_id: str
    content: str
    score: float
    source_code: str
    source_name: str
    attribution_text: str
    license_code: str
    license_url: Optional[str]
    attribution_required: bool
    source_url: Optional[str]
    document_type: str
    category: str
    section_title: Optional[str]
    page_number: Optional[int]


# =============================================================================
# Database Operations
# =============================================================================

class DSFACorpusStore:
    """Database operations for DSFA corpus."""

    def __init__(self, pool: asyncpg.Pool):
        self.pool = pool

    async def register_source(self, source_data: Dict) -> str:
        """Register a DSFA source in the database."""
        async with self.pool.acquire() as conn:
            # Check if source already exists
            existing = await conn.fetchval(
                "SELECT id FROM dsfa_sources WHERE source_code = $1",
                source_data["source_code"]
            )
            if existing:
                # Update existing source
                await conn.execute("""
                    UPDATE dsfa_sources SET
                        name = $2,
                        full_name = $3,
                        organization = $4,
                        source_url = $5,
                        eur_lex_celex = $6,
                        license_code = $7,
                        license_url = $8,
                        attribution_required = $9,
                        attribution_text = $10,
                        document_type = $11,
                        language = $12,
                        updated_at = NOW()
                    WHERE source_code = $1
                """,
                    source_data["source_code"],
                    source_data["name"],
                    source_data.get("full_name"),
                    source_data.get("organization"),
                    source_data.get("source_url"),
                    source_data.get("eur_lex_celex"),
                    source_data["license_code"],
                    source_data.get("license_url"),
                    LICENSE_REGISTRY.get(source_data["license_code"], {}).get("attribution_required", True),
                    source_data["attribution_text"],
                    source_data.get("document_type"),
                    source_data.get("language", "de")
                )
                return str(existing)
            else:
                # Insert new source
                source_id = await conn.fetchval("""
                    INSERT INTO dsfa_sources (
                        source_code, name, full_name, organization, source_url,
                        eur_lex_celex, license_code, license_url, attribution_required,
                        attribution_text, document_type, language
                    ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
                    RETURNING id
                """,
                    source_data["source_code"],
                    source_data["name"],
                    source_data.get("full_name"),
                    source_data.get("organization"),
                    source_data.get("source_url"),
                    source_data.get("eur_lex_celex"),
                    source_data["license_code"],
                    source_data.get("license_url"),
                    LICENSE_REGISTRY.get(source_data["license_code"], {}).get("attribution_required", True),
                    source_data["attribution_text"],
                    source_data.get("document_type"),
                    source_data.get("language", "de")
                )
                return str(source_id)

    async def get_source_by_code(self, source_code: str) -> Optional[Dict]:
        """Get source by its code."""
        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(
                "SELECT * FROM dsfa_sources WHERE source_code = $1",
                source_code
            )
            if row:
                return dict(row)
            return None

    async def list_sources(self) -> List[Dict]:
        """List all registered sources."""
        async with self.pool.acquire() as conn:
            rows = await conn.fetch(
                "SELECT * FROM dsfa_sources ORDER BY source_code"
            )
            return [dict(row) for row in rows]

    async def create_document(
        self,
        source_id: str,
        title: str,
        file_name: Optional[str] = None,
        file_type: Optional[str] = None,
        minio_path: Optional[str] = None,
        original_url: Optional[str] = None,
        metadata: Optional[Dict] = None
    ) -> str:
        """Create a document record."""
        import json
        metadata_json = json.dumps(metadata or {})
        async with self.pool.acquire() as conn:
            doc_id = await conn.fetchval("""
                INSERT INTO dsfa_documents (
                    source_id, title, file_name, file_type, minio_path,
                    original_url, metadata
                ) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
                RETURNING id
            """,
                uuid.UUID(source_id),
                title,
                file_name,
                file_type,
                minio_path,
                original_url,
                metadata_json
            )
            return str(doc_id)

    async def create_chunk(
        self,
        document_id: str,
        source_id: str,
        content: str,
        chunk_index: int,
        section_title: Optional[str] = None,
        page_number: Optional[int] = None,
        category: Optional[str] = None,
        qdrant_point_id: Optional[str] = None,
        metadata: Optional[Dict] = None
    ) -> str:
        """Create a chunk record."""
        import json
        content_hash = hashlib.sha256(content.encode()).hexdigest()

        async with self.pool.acquire() as conn:
            chunk_id = await conn.fetchval("""
                INSERT INTO dsfa_document_chunks (
                    document_id, source_id, content, content_hash, chunk_index,
                    section_title, page_number, category, qdrant_point_id, metadata
                ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb)
                RETURNING id
            """,
                uuid.UUID(document_id),
                uuid.UUID(source_id),
                content,
                content_hash,
                chunk_index,
                section_title,
                page_number,
                category,
                qdrant_point_id,
                json.dumps(metadata or {})
            )
            return str(chunk_id)

    async def get_chunk_with_attribution(self, chunk_id: str) -> Optional[Dict]:
        """Get a chunk with full source attribution."""
        async with self.pool.acquire() as conn:
            row = await conn.fetchrow("""
                SELECT * FROM dsfa_chunk_with_attribution
                WHERE chunk_id = $1
            """, uuid.UUID(chunk_id))
            if row:
                return dict(row)
            return None

    async def get_source_stats(self) -> List[Dict]:
        """Get aggregated stats per source."""
        async with self.pool.acquire() as conn:
            rows = await conn.fetch("SELECT * FROM dsfa_source_stats")
            return [dict(row) for row in rows]

    async def update_document_indexed(self, document_id: str, chunks_count: int):
        """Update document with indexing information."""
        async with self.pool.acquire() as conn:
            await conn.execute("""
                UPDATE dsfa_documents
                SET chunks_generated = $2,
                    last_indexed_at = NOW(),
                    text_extracted = true
                WHERE id = $1
            """, uuid.UUID(document_id), chunks_count)


# =============================================================================
# Qdrant Operations
# =============================================================================

class DSFAQdrantService:
    """Qdrant operations for DSFA corpus."""

    def __init__(self, url: str = None):
        self.url = url or QDRANT_URL
        self._client = None

    @property
    def client(self) -> QdrantClient:
        if self._client is None:
            self._client = QdrantClient(url=self.url, check_compatibility=False)
        return self._client

    async def ensure_collection(self) -> bool:
        """Ensure DSFA collection exists."""
        try:
            collections = self.client.get_collections().collections
            collection_names = [c.name for c in collections]

            if DSFA_COLLECTION not in collection_names:
                self.client.create_collection(
                    collection_name=DSFA_COLLECTION,
                    vectors_config=VectorParams(
                        size=VECTOR_SIZE,
                        distance=Distance.COSINE
                    )
                )
                print(f"Created collection: {DSFA_COLLECTION}")
            return True
        except Exception as e:
            print(f"Error ensuring collection: {e}")
            return False

    async def index_chunks(
        self,
        chunks: List[Dict],
        embeddings: List[List[float]]
    ) -> int:
        """Index chunks into Qdrant."""
        if not chunks or not embeddings:
            return 0

        points = []
        for chunk, embedding in zip(chunks, embeddings):
            point_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, chunk["chunk_id"]))

            payload = DSFAChunkPayload(
                chunk_id=chunk["chunk_id"],
                document_id=chunk["document_id"],
                source_id=chunk["source_id"],
                content=chunk["content"],
                section_title=chunk.get("section_title"),
                source_code=chunk["source_code"],
                source_name=chunk["source_name"],
                attribution_text=chunk["attribution_text"],
                license_code=chunk["license_code"],
                attribution_required=chunk.get("attribution_required", True),
                document_type=chunk.get("document_type", ""),
                category=chunk.get("category", ""),
                language=chunk.get("language", "de"),
                page_number=chunk.get("page_number")
            )

            points.append(
                PointStruct(
                    id=point_id,
                    vector=embedding,
                    payload=asdict(payload)
                )
            )

        self.client.upsert(collection_name=DSFA_COLLECTION, points=points)
        return len(points)

    async def search(
        self,
        query_embedding: List[float],
        source_codes: Optional[List[str]] = None,
        document_types: Optional[List[str]] = None,
        categories: Optional[List[str]] = None,
        limit: int = 10
    ) -> List[Dict]:
        """Search DSFA corpus with filters."""
        must_conditions = []

        if source_codes:
            for code in source_codes:
                must_conditions.append(
                    FieldCondition(key="source_code", match=MatchValue(value=code))
                )

        if document_types:
            for dtype in document_types:
                must_conditions.append(
                    FieldCondition(key="document_type", match=MatchValue(value=dtype))
                )

        if categories:
            for cat in categories:
                must_conditions.append(
                    FieldCondition(key="category", match=MatchValue(value=cat))
                )

        query_filter = Filter(must=must_conditions) if must_conditions else None

        # Use query_points for newer qdrant-client API
        results = self.client.query_points(
            collection_name=DSFA_COLLECTION,
            query=query_embedding,
            query_filter=query_filter,
            limit=limit
        )

        return [
            {
                "id": str(r.id),
                "score": r.score,
                **r.payload
            }
            for r in results.points
        ]

    async def get_stats(self) -> Dict:
        """Get collection statistics."""
        try:
            info = self.client.get_collection(DSFA_COLLECTION)
            return {
                "collection": DSFA_COLLECTION,
                "vectors_count": info.vectors_count,
                "points_count": info.points_count,
                "status": info.status.value
            }
        except Exception as e:
            return {"error": str(e), "collection": DSFA_COLLECTION}


# =============================================================================
# Chunking Functions
# =============================================================================

def chunk_text_recursive(text: str, max_size: int = 1000, overlap: int = 200) -> List[Dict]:
    """Recursively chunk text with overlap."""
    chunks = []
    start = 0

    while start < len(text):
        end = min(start + max_size, len(text))

        # Find a good break point (sentence end, paragraph)
        if end < len(text):
            for sep in ["\n\n", "\n", ". ", ", ", " "]:
                last_sep = text[start:end].rfind(sep)
                if last_sep > max_size // 2:
                    end = start + last_sep + len(sep)
                    break

        chunk_text = text[start:end].strip()
        if chunk_text:
            chunks.append({
                "content": chunk_text,
                "start_char": start,
                "end_char": end
            })

        start = end - overlap if end < len(text) else len(text)

    return chunks


def chunk_by_sections(text: str, markers: List[str], max_size: int = 1500, overlap: int = 200) -> List[Dict]:
    """Chunk text by section markers."""
    chunks = []
    pattern = "|".join(f"({m})" for m in markers)

    # Find all section starts
    matches = list(re.finditer(pattern, text, re.IGNORECASE | re.MULTILINE))

    if not matches:
        return chunk_text_recursive(text, max_size, overlap)

    for i, match in enumerate(matches):
        start = match.start()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)

        section_text = text[start:end].strip()
        section_title = match.group(0).strip()

        if len(section_text) > max_size:
            sub_chunks = chunk_text_recursive(section_text, max_size, overlap)
            for j, sub in enumerate(sub_chunks):
                chunks.append({
                    "content": sub["content"],
                    "section_title": section_title if j == 0 else f"{section_title} (cont.)",
                    "start_char": start + sub["start_char"],
                    "end_char": start + sub["end_char"]
                })
        else:
            chunks.append({
                "content": section_text,
                "section_title": section_title,
                "start_char": start,
                "end_char": end
            })

    return chunks


def chunk_by_list_items(text: str, markers: List[str], max_size: int = 800) -> List[Dict]:
    """Chunk text by list item markers."""
    chunks = []
    pattern = "|".join(f"({m})" for m in markers)

    lines = text.split("\n")
    current_item = ""
    current_start = 0

    for i, line in enumerate(lines):
        if re.match(pattern, line.strip()):
            if current_item.strip():
                chunks.append({
                    "content": current_item.strip(),
                    "start_char": current_start,
                    "end_char": current_start + len(current_item)
                })
            current_item = line
            current_start = sum(len(lines[j]) + 1 for j in range(i))
        else:
            current_item += "\n" + line

    if current_item.strip():
        chunks.append({
            "content": current_item.strip(),
            "start_char": current_start,
            "end_char": current_start + len(current_item)
        })

    return chunks


def chunk_document(text: str, source_code: str) -> List[Dict]:
    """Chunk document using appropriate strategy for source type."""
    config = DSFA_CHUNK_CONFIG.get(source_code, DSFA_CHUNK_CONFIG["DEFAULT"])

    if source_code.endswith("_MUSS_PUBLIC") or source_code.endswith("_MUSS_PRIVATE"):
        config = DSFA_CHUNK_CONFIG["MUSS_LISTEN"]

    if config["strategy"] == "section_based":
        return chunk_by_sections(
            text,
            config["section_markers"],
            config["max_chunk_size"],
            config["overlap"]
        )
    elif config["strategy"] == "list_item":
        return chunk_by_list_items(
            text,
            config["list_markers"],
            config["max_chunk_size"]
        )
    else:
        return chunk_text_recursive(
            text,
            config["max_chunk_size"],
            config["overlap"]
        )


# =============================================================================
# Attribution Functions
# =============================================================================

def generate_attribution_notice(results: List[DSFASearchResult]) -> str:
    """Generate combined attribution notice for all used sources."""
    from collections import defaultdict

    by_license = defaultdict(list)
    for r in results:
        by_license[r.license_code].append(r)

    notices = []
    for license_code, items in by_license.items():
        license_info = LICENSE_REGISTRY.get(license_code, {})
        if license_info.get("attribution_required", True):
            sources = ", ".join(set(r.source_name for r in items))
            license_name = license_info.get("name", license_code)
            notices.append(f"• {sources} - {license_name}")

    if notices:
        return "Quellennachweis:\n" + "\n".join(notices)
    return ""


def get_license_label(license_code: str) -> str:
    """Get human-readable license label."""
    license_info = LICENSE_REGISTRY.get(license_code, {})
    return license_info.get("name", license_code)


# =============================================================================
# Main Functions
# =============================================================================

async def init_dsfa_tables(pool: asyncpg.Pool):
    """Initialize DSFA tables by running migration."""
    migration_path = os.path.join(
        os.path.dirname(__file__),
        "migrations",
        "003_dsfa_rag_tables.sql"
    )

    async with pool.acquire() as conn:
        with open(migration_path, "r") as f:
            await conn.execute(f.read())

    print("DSFA tables initialized successfully")


async def register_all_sources(pool: asyncpg.Pool):
    """Register all DSFA sources in the database (skips migrated sources)."""
    store = DSFACorpusStore(pool)

    registered = 0
    skipped = 0
    for source in DSFA_SOURCES:
        if source.get("migrated_to"):
            print(f"Skipping migrated source: {source['source_code']} -> {source['migrated_to']}")
            skipped += 1
            continue
        source_id = await store.register_source(source)
        print(f"Registered source: {source['source_code']} -> {source_id}")
        registered += 1

    print(f"\nTotal sources registered: {registered} (skipped {skipped} migrated)")


async def get_ingestion_status(pool: asyncpg.Pool):
    """Get current ingestion status."""
    store = DSFACorpusStore(pool)
    qdrant = DSFAQdrantService()

    print("\n=== DSFA Corpus Status ===\n")

    # PostgreSQL stats
    stats = await store.get_source_stats()
    print("PostgreSQL Sources:")
    print("-" * 80)
    print(f"{'Source Code':<25} {'Documents':>10} {'Chunks':>10} {'Last Indexed':<20}")
    print("-" * 80)

    total_docs = 0
    total_chunks = 0
    for s in stats:
        total_docs += s.get("document_count", 0)
        total_chunks += s.get("chunk_count", 0)
        last_indexed = s.get("last_indexed_at")
        last_indexed_str = last_indexed.strftime("%Y-%m-%d %H:%M") if last_indexed else "Never"
        print(f"{s['source_code']:<25} {s.get('document_count', 0):>10} {s.get('chunk_count', 0):>10} {last_indexed_str:<20}")

    print("-" * 80)
    print(f"{'TOTAL':<25} {total_docs:>10} {total_chunks:>10}")

    # Qdrant stats
    print("\nQdrant Collection:")
    qdrant_stats = await qdrant.get_stats()
    if "error" in qdrant_stats:
        print(f"  Error: {qdrant_stats['error']}")
    else:
        print(f"  Collection: {qdrant_stats['collection']}")
        print(f"  Points: {qdrant_stats['points_count']}")
        print(f"  Status: {qdrant_stats['status']}")


async def main():
    """Main entry point."""
    parser = argparse.ArgumentParser(description="DSFA Corpus Ingestion Pipeline")
    parser.add_argument("--init-sources", action="store_true", help="Register all sources")
    parser.add_argument("--init-tables", action="store_true", help="Initialize database tables")
    parser.add_argument("--ingest", type=str, help="Ingest specific source by code")
    parser.add_argument("--ingest-all", action="store_true", help="Ingest all sources")
    parser.add_argument("--status", action="store_true", help="Show ingestion status")
    parser.add_argument("--init-qdrant", action="store_true", help="Initialize Qdrant collection")

    args = parser.parse_args()

    # Connect to database
    pool = await asyncpg.create_pool(DATABASE_URL)

    try:
        if args.init_tables:
            await init_dsfa_tables(pool)

        if args.init_sources:
            await register_all_sources(pool)

        if args.init_qdrant:
            qdrant = DSFAQdrantService()
            await qdrant.ensure_collection()
            print(f"Qdrant collection {DSFA_COLLECTION} initialized")

        if args.status:
            await get_ingestion_status(pool)

        if args.ingest:
            print(f"Ingesting source: {args.ingest}")
            # TODO: Implement document ingestion

        if args.ingest_all:
            print("Ingesting all sources...")
            # TODO: Implement bulk ingestion

    finally:
        await pool.close()


if __name__ == "__main__":
    asyncio.run(main())
