Benjamin_Boenisch/breakpilot-compliance
1
0
Fork 0
Code Issues 24 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Semantic Qdrant search — embed query via bge-m3, vector search in local Qdrant

Browse Source 
Replaces scroll+filter approach with proper semantic search:
1. Embed query via bp-core-embedding-service (bge-m3, 1024 dim)
2. Vector search in Qdrant (bp_compliance_datenschutz + bp_compliance_gesetze)
3. Sort by cosine similarity score
4. No API key needed — local Qdrant on Mac Mini

Falls back gracefully: SDK first, then semantic Qdrant, then empty.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-05-06 14:46:06 +02:00
parent 7b8440191e
commit f4374cfe8d
1 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend-compliance/compliance/services/rag_document_checker.py
+34 -34
View File
@@ -115,56 +115,56 @@ async def _search_via_sdk(regulations: list[str], top_k: int) -> list[dict]:
return []
async def _search_via_qdrant(regulations: list[str], top_k: int) -> list[dict]:
"""Search directly in local Qdrant — scroll with payload filter."""
try:
all_results = []
collections = ["bp_compliance_datenschutz", "bp_compliance_gesetze"]
EMBEDDING_URL = os.getenv("EMBEDDING_SERVICE_URL", "http://bp-core-embedding-service:8087")
for collection in collections:
# Scroll through points, filter by section/regulation matching
async def _search_via_qdrant(regulations: list[str], top_k: int) -> list[dict]:
"""Semantic search in local Qdrant via embedding + vector search."""
try:
# Step 1: Embed the query
query_text = " ".join(regulations[:3]) + " Pflichtangaben Anforderungen"
async with httpx.AsyncClient(timeout=15.0) as client:
emb_resp = await client.post(f"{EMBEDDING_URL}/embed", json={"texts": [query_text]})
if emb_resp.status_code != 200:
logger.warning("Embedding failed: %d", emb_resp.status_code)
return []
vector = emb_resp.json().get("embeddings", [[]])[0]
if not vector:
return []
# Step 2: Search Qdrant with vector
all_results = []
for collection in ["bp_compliance_datenschutz", "bp_compliance_gesetze"]:
async with httpx.AsyncClient(timeout=10.0) as client:
resp = await client.post(f"{QDRANT_URL}/collections/{collection}/points/scroll", json={
"limit": 100, # Fetch more, filter client-side
resp = await client.post(f"{QDRANT_URL}/collections/{collection}/points/search", json={
"vector": vector,
"limit": top_k,
"with_payload": True,
"with_vector": False,
})
if resp.status_code != 200:
continue
data = resp.json()
for point in data.get("result", {}).get("points", []):
for point in data.get("result", []):
payload = point.get("payload", {})
chunk = payload.get("chunk_text", "")
section = payload.get("section", "")
category = payload.get("category", "")
reg_id = payload.get("regulation_id", "")
section_title = payload.get("section_title", "")
if not chunk or len(chunk) < 50:
continue
all_results.append({
"text": chunk[:500],
"regulation": payload.get("regulation_id", "") or payload.get("section", ""),
"article": payload.get("section", ""),
"score": point.get("score", 0.0),
})
# Match against regulation keywords
searchable = f"{section} {category} {reg_id} {section_title} {chunk[:200]}".lower()
matched = any(
kw.lower() in searchable
for r in regulations
for kw in [r, r.replace("Art. ", "Article "), r.replace("§", "")]
)
if matched:
all_results.append({
"text": chunk[:500],
"regulation": reg_id or section or category,
"article": section,
"score": 0.5,
})
logger.info("Qdrant direct search: found %d controls from %d collections",
len(all_results), len(collections))
# Sort by score descending
all_results.sort(key=lambda x: x["score"], reverse=True)
logger.info("Qdrant semantic search: found %d results", len(all_results))
return all_results[:top_k]
except Exception as e:
logger.warning("Direct Qdrant search failed: %s", e)
logger.warning("Qdrant semantic search failed: %s", e)
return []