feat(pipeline): structural metadata end-to-end (Blocks D2-D4)

D2: RAG service stores section/section_title/paragraph/paragraph_num/page
from embedding service chunks_with_metadata into Qdrant payloads.

D3: Control generator prefers section > article > section_title from
Qdrant, adds page to source_citation and generation_metadata.

D4: Validated with real BGB §§ 312-312k text. Found and fixed critical
bug where Phase 3 overlap destroyed the [§ ...] section prefix, causing
only the first chunk per document to have metadata. All subsequent
chunks lost section info.

Also fixes pre-existing lint issues (unused imports, ambiguous variable
names, duplicate dict key, bare except).

456 tests passing (58 embedding + 387 pipeline + 11 rag-service).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

control-pipeline/services/control_generator.py

@@ -25,8 +25,7 @@ import re
 import uuid
 from collections import defaultdict
 from dataclasses import dataclass, field, asdict
-from datetime import datetime, timezone
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, Optional
 
 import httpx
 from pydantic import BaseModel
@@ -34,7 +33,7 @@ from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from .rag_client import ComplianceRAGClient, RAGSearchResult, get_rag_client
-from .similarity_detector import check_similarity, SimilarityReport
+from .similarity_detector import check_similarity
 
 logger = logging.getLogger(__name__)
 
@@ -1019,11 +1018,12 @@ class ControlGeneratorPipeline:
                         regulation_name=reg_name,
                         regulation_short=reg_short,
                         category=payload.get("category", "") or payload.get("data_type", ""),
-                        article=payload.get("article", "") or payload.get("section_title", "") or payload.get("section", ""),
+                        article=payload.get("section", "") or payload.get("article", "") or payload.get("section_title", ""),
                         paragraph=payload.get("paragraph", ""),
                         source_url=payload.get("source_url", "") or payload.get("source", "") or payload.get("url", ""),
                         score=0.0,
                         collection=collection,
+                        page=payload.get("page"),
                     )
                     all_results.append(chunk)
                     collection_new += 1
@@ -1127,6 +1127,7 @@ Quelle: {chunk.regulation_name} ({chunk.regulation_code}), {chunk.article}"""
             "source": canonical_source,
             "article": effective_article,
             "paragraph": effective_paragraph,
+            "page": chunk.page,
             "license": license_info.get("license", ""),
             "source_type": license_info.get("source_type", "law"),
             "url": chunk.source_url or "",
@@ -1141,6 +1142,7 @@ Quelle: {chunk.regulation_name} ({chunk.regulation_code}), {chunk.article}"""
             "source_regulation": chunk.regulation_code,
             "source_article": effective_article,
             "source_paragraph": effective_paragraph,
+            "source_page": chunk.page,
         }
         return control
 
@@ -1194,6 +1196,7 @@ Quelle: {chunk.regulation_name}, {chunk.article}"""
             "source": canonical_source,
             "article": effective_article,
             "paragraph": effective_paragraph,
+            "page": chunk.page,
             "license": license_info.get("license", ""),
             "license_notice": attribution,
             "source_type": license_info.get("source_type", "standard"),
@@ -1209,6 +1212,7 @@ Quelle: {chunk.regulation_name}, {chunk.article}"""
             "source_regulation": chunk.regulation_code,
             "source_article": effective_article,
             "source_paragraph": effective_paragraph,
+            "source_page": chunk.page,
         }
         return control
 
@@ -1368,6 +1372,7 @@ Gib ein JSON-Array zurueck mit GENAU {len(chunks)} Elementen. Fuer Chunks ohne A
                     "source": canonical_source,
                     "article": effective_article,
                     "paragraph": effective_paragraph,
+                    "page": chunk.page,
                     "license": lic.get("license", ""),
                     "license_notice": lic.get("attribution", ""),
                     "source_type": lic.get("source_type", "law"),
@@ -1384,6 +1389,7 @@ Gib ein JSON-Array zurueck mit GENAU {len(chunks)} Elementen. Fuer Chunks ohne A
                 "source_regulation": chunk.regulation_code,
                 "source_article": effective_article,
                 "source_paragraph": effective_paragraph,
+                "source_page": chunk.page,
                 "batch_size": len(chunks),
                 "document_grouped": same_doc,
             }
@@ -1479,14 +1485,14 @@ Gib ein JSON-Array zurueck mit GENAU {len(chunks)} Elementen. Fuer Aspekte ohne
     ) -> list[Optional[GeneratedControl]]:
         """Process a batch of (chunk, license_info) through stages 3-5."""
         # Split by license rule: Rule 1+2 → structure, Rule 3 → reform
-        structure_items = [(c, l) for c, l in batch_items if l["rule"] in (1, 2)]
-        reform_items = [(c, l) for c, l in batch_items if l["rule"] == 3]
+        structure_items = [(c, lic) for c, lic in batch_items if lic["rule"] in (1, 2)]
+        reform_items = [(c, lic) for c, lic in batch_items if lic["rule"] == 3]
 
         all_controls: dict[int, Optional[GeneratedControl]] = {}
 
         if structure_items:
             s_chunks = [c for c, _ in structure_items]
-            s_lics = [l for _, l in structure_items]
+            s_lics = [lic for _, lic in structure_items]
             try:
                 s_controls = await self._structure_batch(s_chunks, s_lics)
             except Exception as e: