fix: RAG checker falls back to local Qdrant when Go SDK returns 401
Go SDK points to external Qdrant (qdrant-dev.breakpilot.ai) with expired API key. Fallback: search directly in local Qdrant (bp-core-qdrant:6333) which has all collections: bp_compliance_datenschutz, bp_compliance_gesetze, atomic_controls_dedup. Search strategy: 1. Try Go SDK RAG endpoint (preferred, has embedding-based search) 2. Fallback: Qdrant scroll with text-based regulation filter Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -26,6 +26,7 @@ logger = logging.getLogger(__name__)
|
|||||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://host.docker.internal:11434")
|
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://host.docker.internal:11434")
|
||||||
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3.5:35b-a3b")
|
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3.5:35b-a3b")
|
||||||
SDK_URL = os.getenv("SDK_URL", "http://ai-compliance-sdk:8090")
|
SDK_URL = os.getenv("SDK_URL", "http://ai-compliance-sdk:8090")
|
||||||
|
QDRANT_URL = os.getenv("QDRANT_INTERNAL_URL", "http://bp-core-qdrant:6333")
|
||||||
|
|
||||||
# Document type → Regulation keywords for RAG filtering
|
# Document type → Regulation keywords for RAG filtering
|
||||||
DOC_TYPE_REGULATIONS = {
|
DOC_TYPE_REGULATIONS = {
|
||||||
@@ -80,36 +81,80 @@ async def _search_relevant_controls(
|
|||||||
regulations: list[str],
|
regulations: list[str],
|
||||||
top_k: int = 10,
|
top_k: int = 10,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Search RAG for controls relevant to this document."""
|
"""Search for relevant controls — tries Go SDK first, falls back to direct Qdrant."""
|
||||||
try:
|
# Try Go SDK RAG endpoint first
|
||||||
# Use the first regulation as primary query, rest as context
|
controls = await _search_via_sdk(regulations, top_k)
|
||||||
query = f"{regulations[0]} Anforderungen Pflichtangaben"
|
if controls:
|
||||||
|
return controls
|
||||||
|
|
||||||
|
# Fallback: search directly in Qdrant (local Mac Mini)
|
||||||
|
controls = await _search_via_qdrant(regulations, top_k)
|
||||||
|
return controls
|
||||||
|
|
||||||
|
|
||||||
|
async def _search_via_sdk(regulations: list[str], top_k: int) -> list[dict]:
|
||||||
|
"""Search via Go SDK RAG endpoint."""
|
||||||
|
try:
|
||||||
|
query = f"{regulations[0]} Anforderungen Pflichtangaben"
|
||||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||||
resp = await client.post(f"{SDK_URL}/sdk/v1/rag/search", json={
|
resp = await client.post(f"{SDK_URL}/sdk/v1/rag/search", json={
|
||||||
"query": query,
|
"query": query,
|
||||||
"collection": "bp_compliance_datenschutz",
|
"collection": "bp_compliance_datenschutz",
|
||||||
"top_k": top_k,
|
"top_k": top_k,
|
||||||
})
|
})
|
||||||
|
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
logger.warning("RAG search returned %d", resp.status_code)
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
controls = []
|
return [{
|
||||||
for r in data.get("results", []):
|
|
||||||
controls.append({
|
|
||||||
"text": r.get("text", ""),
|
"text": r.get("text", ""),
|
||||||
"regulation": r.get("regulation_code", "") or r.get("regulation_short", ""),
|
"regulation": r.get("regulation_code", "") or r.get("regulation_short", ""),
|
||||||
"article": r.get("article", ""),
|
"article": r.get("article", ""),
|
||||||
"score": r.get("score", 0.0),
|
"score": r.get("score", 0.0),
|
||||||
|
} for r in data.get("results", [])]
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
async def _search_via_qdrant(regulations: list[str], top_k: int) -> list[dict]:
|
||||||
|
"""Search directly in local Qdrant — keyword scroll with filter."""
|
||||||
|
try:
|
||||||
|
# Search in multiple collections
|
||||||
|
all_results = []
|
||||||
|
for collection in ["bp_compliance_datenschutz", "bp_compliance_gesetze", "atomic_controls_dedup"]:
|
||||||
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||||
|
# Scroll with text filter (Qdrant scroll endpoint)
|
||||||
|
resp = await client.post(f"{QDRANT_URL}/collections/{collection}/points/scroll", json={
|
||||||
|
"limit": top_k,
|
||||||
|
"with_payload": True,
|
||||||
|
"with_vector": False,
|
||||||
|
})
|
||||||
|
if resp.status_code != 200:
|
||||||
|
continue
|
||||||
|
data = resp.json()
|
||||||
|
for point in data.get("result", {}).get("points", []):
|
||||||
|
payload = point.get("payload", {})
|
||||||
|
text = payload.get("text", "") or payload.get("content", "") or payload.get("chunk_text", "")
|
||||||
|
if not text:
|
||||||
|
continue
|
||||||
|
# Filter: only keep results that mention our regulations
|
||||||
|
text_lower = text.lower()
|
||||||
|
reg_match = any(
|
||||||
|
r.lower().replace("§", "").replace("art.", "art").strip() in text_lower
|
||||||
|
for r in regulations
|
||||||
|
)
|
||||||
|
if reg_match and len(text) > 50:
|
||||||
|
all_results.append({
|
||||||
|
"text": text[:500],
|
||||||
|
"regulation": payload.get("regulation_code", "") or payload.get("regulation_short", ""),
|
||||||
|
"article": payload.get("article", ""),
|
||||||
|
"score": 0.5,
|
||||||
})
|
})
|
||||||
|
|
||||||
return controls
|
logger.info("Qdrant direct search: found %d controls", len(all_results))
|
||||||
|
return all_results[:top_k]
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("RAG control search failed: %s", e)
|
logger.warning("Direct Qdrant search failed: %s", e)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user