From f4374cfe8d9af665d024d511475b04393dd03423 Mon Sep 17 00:00:00 2001
From: Benjamin Admin <benjaminadmin@MacBook-Pro.local>
Date: Wed, 6 May 2026 14:46:06 +0200
Subject: [PATCH] =?UTF-8?q?feat:=20Semantic=20Qdrant=20search=20=E2=80=94?=
 =?UTF-8?q?=20embed=20query=20via=20bge-m3,=20vector=20search=20in=20local?=
 =?UTF-8?q?=20Qdrant?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replaces scroll+filter approach with proper semantic search:
1. Embed query via bp-core-embedding-service (bge-m3, 1024 dim)
2. Vector search in Qdrant (bp_compliance_datenschutz + bp_compliance_gesetze)
3. Sort by cosine similarity score
4. No API key needed — local Qdrant on Mac Mini

Falls back gracefully: SDK first, then semantic Qdrant, then empty.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../services/rag_document_checker.py          | 68 +++++++++----------
 1 file changed, 34 insertions(+), 34 deletions(-)

diff --git a/backend-compliance/compliance/services/rag_document_checker.py b/backend-compliance/compliance/services/rag_document_checker.py
index f51bc25..2ae45e0 100644
--- a/backend-compliance/compliance/services/rag_document_checker.py
+++ b/backend-compliance/compliance/services/rag_document_checker.py
@@ -115,56 +115,56 @@ async def _search_via_sdk(regulations: list[str], top_k: int) -> list[dict]:
         return []
 
 
-async def _search_via_qdrant(regulations: list[str], top_k: int) -> list[dict]:
-    """Search directly in local Qdrant — scroll with payload filter."""
-    try:
-        all_results = []
-        collections = ["bp_compliance_datenschutz", "bp_compliance_gesetze"]
+EMBEDDING_URL = os.getenv("EMBEDDING_SERVICE_URL", "http://bp-core-embedding-service:8087")
 
-        for collection in collections:
-            # Scroll through points, filter by section/regulation matching
+
+async def _search_via_qdrant(regulations: list[str], top_k: int) -> list[dict]:
+    """Semantic search in local Qdrant via embedding + vector search."""
+    try:
+        # Step 1: Embed the query
+        query_text = " ".join(regulations[:3]) + " Pflichtangaben Anforderungen"
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            emb_resp = await client.post(f"{EMBEDDING_URL}/embed", json={"texts": [query_text]})
+        if emb_resp.status_code != 200:
+            logger.warning("Embedding failed: %d", emb_resp.status_code)
+            return []
+
+        vector = emb_resp.json().get("embeddings", [[]])[0]
+        if not vector:
+            return []
+
+        # Step 2: Search Qdrant with vector
+        all_results = []
+        for collection in ["bp_compliance_datenschutz", "bp_compliance_gesetze"]:
             async with httpx.AsyncClient(timeout=10.0) as client:
-                resp = await client.post(f"{QDRANT_URL}/collections/{collection}/points/scroll", json={
-                    "limit": 100,  # Fetch more, filter client-side
+                resp = await client.post(f"{QDRANT_URL}/collections/{collection}/points/search", json={
+                    "vector": vector,
+                    "limit": top_k,
                     "with_payload": True,
-                    "with_vector": False,
                 })
             if resp.status_code != 200:
                 continue
 
             data = resp.json()
-            for point in data.get("result", {}).get("points", []):
+            for point in data.get("result", []):
                 payload = point.get("payload", {})
                 chunk = payload.get("chunk_text", "")
-                section = payload.get("section", "")
-                category = payload.get("category", "")
-                reg_id = payload.get("regulation_id", "")
-                section_title = payload.get("section_title", "")
-
                 if not chunk or len(chunk) < 50:
                     continue
+                all_results.append({
+                    "text": chunk[:500],
+                    "regulation": payload.get("regulation_id", "") or payload.get("section", ""),
+                    "article": payload.get("section", ""),
+                    "score": point.get("score", 0.0),
+                })
 
-                # Match against regulation keywords
-                searchable = f"{section} {category} {reg_id} {section_title} {chunk[:200]}".lower()
-                matched = any(
-                    kw.lower() in searchable
-                    for r in regulations
-                    for kw in [r, r.replace("Art. ", "Article "), r.replace("§", "")]
-                )
-                if matched:
-                    all_results.append({
-                        "text": chunk[:500],
-                        "regulation": reg_id or section or category,
-                        "article": section,
-                        "score": 0.5,
-                    })
-
-        logger.info("Qdrant direct search: found %d controls from %d collections",
-                     len(all_results), len(collections))
+        # Sort by score descending
+        all_results.sort(key=lambda x: x["score"], reverse=True)
+        logger.info("Qdrant semantic search: found %d results", len(all_results))
         return all_results[:top_k]
 
     except Exception as e:
-        logger.warning("Direct Qdrant search failed: %s", e)
+        logger.warning("Qdrant semantic search failed: %s", e)
         return []