feat(rag): optimize RAG pipeline — JSON-Mode, CoT, Hybrid Search, Re-Ranking, Cross-Reg Dedup, chunk 1024

Phase 1 (LLM Quality):
- Add format=json to all Ollama payloads (obligation_extractor, control_generator, citation_backfill)
- Add Chain-of-Thought analysis steps to Pass 0a/0b system prompts

Phase 2 (Retrieval Quality):
- Hybrid search via Qdrant Query API with RRF fusion + automatic text index (legal_rag.go)
- Fallback to dense-only search if Query API unavailable
- Cross-encoder re-ranking with BGE Reranker v2 (RERANK_ENABLED=false by default)
- CPU-only PyTorch dependency to keep Docker image small

Phase 3 (Data Layer):
- Cross-regulation dedup pass (threshold 0.95) links controls across regulations
- DedupResult.link_type field distinguishes dedup_merge vs cross_regulation
- Chunk size defaults updated 512/50 → 1024/128 for new ingestions only
- Existing collections and controls are NOT affected

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

backend-compliance/tests/test_decomposition_pass.py

@@ -25,7 +25,11 @@ from compliance.services.decomposition_pass import (
     AtomicControlCandidate,
     quality_gate,
     passes_quality_gate,
+    classify_obligation_type,
     _NORMATIVE_RE,
+    _PFLICHT_RE,
+    _EMPFEHLUNG_RE,
+    _KANN_RE,
     _RATIONALE_RE,
     _TEST_RE,
     _REPORTING_RE,
@@ -176,7 +180,7 @@ class TestQualityGate:
     def test_rationale_detected(self):
         oc = ObligationCandidate(
             parent_control_uuid="uuid-1",
-            obligation_text="Schwache Passwörter können zu Risiken führen, weil sie leicht zu erraten sind",
+            obligation_text="Dies liegt daran, weil schwache Konfigurationen ein Risiko darstellen",
         )
         flags = quality_gate(oc)
         assert flags["not_rationale"] is False
@@ -228,14 +232,28 @@ class TestQualityGate:
         )
         flags = quality_gate(oc)
         assert flags["has_normative_signal"] is False
+        assert flags["obligation_type"] == "empfehlung"
+
+    def test_obligation_type_in_flags(self):
+        oc = ObligationCandidate(
+            parent_control_uuid="uuid-1",
+            obligation_text="Der Betreiber muss alle Daten verschlüsseln.",
+        )
+        flags = quality_gate(oc)
+        assert flags["obligation_type"] == "pflicht"
 
 
 class TestPassesQualityGate:
-    """Tests for passes_quality_gate function."""
+    """Tests for passes_quality_gate function.
+
+    Note: has_normative_signal is NO LONGER critical — obligations without
+    normative signal are classified as 'empfehlung' instead of being rejected.
+    """
 
     def test_all_critical_pass(self):
         flags = {
             "has_normative_signal": True,
+            "obligation_type": "pflicht",
             "single_action": True,
             "not_rationale": True,
             "not_evidence_only": True,
@@ -244,20 +262,23 @@ class TestPassesQualityGate:
         }
         assert passes_quality_gate(flags) is True
 
-    def test_no_normative_signal_fails(self):
+    def test_no_normative_signal_still_passes(self):
+        """No normative signal no longer causes rejection — classified as empfehlung."""
         flags = {
             "has_normative_signal": False,
+            "obligation_type": "empfehlung",
             "single_action": True,
             "not_rationale": True,
             "not_evidence_only": True,
             "min_length": True,
             "has_parent_link": True,
         }
-        assert passes_quality_gate(flags) is False
+        assert passes_quality_gate(flags) is True
 
     def test_evidence_only_fails(self):
         flags = {
             "has_normative_signal": True,
+            "obligation_type": "pflicht",
             "single_action": True,
             "not_rationale": True,
             "not_evidence_only": False,
@@ -267,9 +288,10 @@ class TestPassesQualityGate:
         assert passes_quality_gate(flags) is False
 
     def test_non_critical_dont_block(self):
-        """single_action and not_rationale are NOT critical — should still pass."""
+        """single_action, not_rationale, has_normative_signal are NOT critical."""
         flags = {
-            "has_normative_signal": True,
+            "has_normative_signal": False,  # Not critical
+            "obligation_type": "empfehlung",
             "single_action": False,  # Not critical
             "not_rationale": False,  # Not critical
             "not_evidence_only": True,
@@ -279,6 +301,42 @@ class TestPassesQualityGate:
         assert passes_quality_gate(flags) is True
 
 
+class TestClassifyObligationType:
+    """Tests for the 3-tier obligation type classification."""
+
+    def test_pflicht_muss(self):
+        assert classify_obligation_type("Der Betreiber muss alle Daten verschlüsseln") == "pflicht"
+
+    def test_pflicht_ist_zu(self):
+        assert classify_obligation_type("Die Meldung ist innerhalb von 72 Stunden zu erstatten") == "pflicht"
+
+    def test_pflicht_shall(self):
+        assert classify_obligation_type("The controller shall implement appropriate measures") == "pflicht"
+
+    def test_empfehlung_soll(self):
+        assert classify_obligation_type("Der Betreiber soll regelmäßige Audits durchführen") == "empfehlung"
+
+    def test_empfehlung_should(self):
+        assert classify_obligation_type("Organizations should implement security controls") == "empfehlung"
+
+    def test_empfehlung_sicherstellen(self):
+        assert classify_obligation_type("Die Verfügbarkeit der Systeme sicherstellen") == "empfehlung"
+
+    def test_kann(self):
+        assert classify_obligation_type("Der Betreiber kann zusätzliche Maßnahmen ergreifen") == "kann"
+
+    def test_kann_may(self):
+        assert classify_obligation_type("The organization may implement optional safeguards") == "kann"
+
+    def test_no_signal_defaults_to_empfehlung(self):
+        assert classify_obligation_type("Regelmäßige Überprüfung der Zugriffsrechte") == "empfehlung"
+
+    def test_pflicht_overrides_empfehlung(self):
+        """If both pflicht and empfehlung signals present, pflicht wins."""
+        txt = "Der Betreiber muss sicherstellen, dass alle Daten verschlüsselt werden"
+        assert classify_obligation_type(txt) == "pflicht"
+
+
 # ---------------------------------------------------------------------------
 # HELPER TESTS
 # ---------------------------------------------------------------------------
@@ -520,6 +578,24 @@ class TestPromptBuilders:
         assert "REGELN" in _PASS0A_SYSTEM_PROMPT
         assert "atomares" in _PASS0B_SYSTEM_PROMPT
 
+    def test_pass0a_prompt_contains_cot_steps(self):
+        """Pass 0a system prompt must include Chain-of-Thought analysis steps."""
+        assert "ANALYSE-SCHRITTE" in _PASS0A_SYSTEM_PROMPT
+        assert "Adressaten" in _PASS0A_SYSTEM_PROMPT
+        assert "Handlung" in _PASS0A_SYSTEM_PROMPT
+        assert "normative Staerke" in _PASS0A_SYSTEM_PROMPT
+        assert "Meldepflicht" in _PASS0A_SYSTEM_PROMPT
+        assert "NICHT im Output" in _PASS0A_SYSTEM_PROMPT
+
+    def test_pass0b_prompt_contains_cot_steps(self):
+        """Pass 0b system prompt must include Chain-of-Thought analysis steps."""
+        assert "ANALYSE-SCHRITTE" in _PASS0B_SYSTEM_PROMPT
+        assert "Anforderung" in _PASS0B_SYSTEM_PROMPT
+        assert "Massnahme" in _PASS0B_SYSTEM_PROMPT
+        assert "Pruefverfahren" in _PASS0B_SYSTEM_PROMPT
+        assert "Nachweis" in _PASS0B_SYSTEM_PROMPT
+        assert "NICHT im Output" in _PASS0B_SYSTEM_PROMPT
+
 
 # ---------------------------------------------------------------------------
 # DECOMPOSITION PASS INTEGRATION TESTS