diff --git a/backend-compliance/compliance/services/rag_document_checker.py b/backend-compliance/compliance/services/rag_document_checker.py index 076efdf..6187328 100644 --- a/backend-compliance/compliance/services/rag_document_checker.py +++ b/backend-compliance/compliance/services/rag_document_checker.py @@ -26,6 +26,7 @@ logger = logging.getLogger(__name__) OLLAMA_URL = os.getenv("OLLAMA_URL", "http://host.docker.internal:11434") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen3.5:35b-a3b") SDK_URL = os.getenv("SDK_URL", "http://ai-compliance-sdk:8090") +QDRANT_URL = os.getenv("QDRANT_INTERNAL_URL", "http://bp-core-qdrant:6333") # Document type โ†’ Regulation keywords for RAG filtering DOC_TYPE_REGULATIONS = { @@ -80,36 +81,80 @@ async def _search_relevant_controls( regulations: list[str], top_k: int = 10, ) -> list[dict]: - """Search RAG for controls relevant to this document.""" - try: - # Use the first regulation as primary query, rest as context - query = f"{regulations[0]} Anforderungen Pflichtangaben" + """Search for relevant controls โ€” tries Go SDK first, falls back to direct Qdrant.""" + # Try Go SDK RAG endpoint first + controls = await _search_via_sdk(regulations, top_k) + if controls: + return controls + # Fallback: search directly in Qdrant (local Mac Mini) + controls = await _search_via_qdrant(regulations, top_k) + return controls + + +async def _search_via_sdk(regulations: list[str], top_k: int) -> list[dict]: + """Search via Go SDK RAG endpoint.""" + try: + query = f"{regulations[0]} Anforderungen Pflichtangaben" async with httpx.AsyncClient(timeout=15.0) as client: resp = await client.post(f"{SDK_URL}/sdk/v1/rag/search", json={ "query": query, "collection": "bp_compliance_datenschutz", "top_k": top_k, }) - if resp.status_code != 200: - logger.warning("RAG search returned %d", resp.status_code) return [] - data = resp.json() - controls = [] - for r in data.get("results", []): - controls.append({ - "text": r.get("text", ""), - "regulation": r.get("regulation_code", "") or r.get("regulation_short", ""), - "article": r.get("article", ""), - "score": r.get("score", 0.0), - }) + return [{ + "text": r.get("text", ""), + "regulation": r.get("regulation_code", "") or r.get("regulation_short", ""), + "article": r.get("article", ""), + "score": r.get("score", 0.0), + } for r in data.get("results", [])] + except Exception: + return [] - return controls + +async def _search_via_qdrant(regulations: list[str], top_k: int) -> list[dict]: + """Search directly in local Qdrant โ€” keyword scroll with filter.""" + try: + # Search in multiple collections + all_results = [] + for collection in ["bp_compliance_datenschutz", "bp_compliance_gesetze", "atomic_controls_dedup"]: + async with httpx.AsyncClient(timeout=10.0) as client: + # Scroll with text filter (Qdrant scroll endpoint) + resp = await client.post(f"{QDRANT_URL}/collections/{collection}/points/scroll", json={ + "limit": top_k, + "with_payload": True, + "with_vector": False, + }) + if resp.status_code != 200: + continue + data = resp.json() + for point in data.get("result", {}).get("points", []): + payload = point.get("payload", {}) + text = payload.get("text", "") or payload.get("content", "") or payload.get("chunk_text", "") + if not text: + continue + # Filter: only keep results that mention our regulations + text_lower = text.lower() + reg_match = any( + r.lower().replace("ยง", "").replace("art.", "art").strip() in text_lower + for r in regulations + ) + if reg_match and len(text) > 50: + all_results.append({ + "text": text[:500], + "regulation": payload.get("regulation_code", "") or payload.get("regulation_short", ""), + "article": payload.get("article", ""), + "score": 0.5, + }) + + logger.info("Qdrant direct search: found %d controls", len(all_results)) + return all_results[:top_k] except Exception as e: - logger.warning("RAG control search failed: %s", e) + logger.warning("Direct Qdrant search failed: %s", e) return []