diff --git a/admin-compliance/app/sdk/agent/_components/ComplianceCheckTab.tsx b/admin-compliance/app/sdk/agent/_components/ComplianceCheckTab.tsx index 1fe3d13..4811d6c 100644 --- a/admin-compliance/app/sdk/agent/_components/ComplianceCheckTab.tsx +++ b/admin-compliance/app/sdk/agent/_components/ComplianceCheckTab.tsx @@ -202,9 +202,9 @@ export function ComplianceCheckTab() { setActiveCheckId(check_id) localStorage.setItem(STORAGE_KEY_CHECK_ID, check_id) - // Poll for results + // Poll for results (max 15 min = 300 polls x 3s) let attempts = 0 - while (attempts < 120) { + while (attempts < 300) { await new Promise(r => setTimeout(r, 3000)) const pollRes = await fetch(`/api/sdk/v1/agent/compliance-check?check_id=${check_id}`) if (!pollRes.ok) { attempts++; continue } @@ -235,9 +235,9 @@ export function ComplianceCheckTab() { } attempts++ } - if (attempts >= 120) { + if (attempts >= 300) { localStorage.removeItem(STORAGE_KEY_CHECK_ID); setActiveCheckId('') - throw new Error('Zeitlimit ueberschritten') + throw new Error('Zeitlimit ueberschritten (15 Min)') } } catch (e) { setError(e instanceof Error ? e.message : 'Unbekannter Fehler') diff --git a/backend-compliance/compliance/services/doc_checks/llm_verify.py b/backend-compliance/compliance/services/doc_checks/llm_verify.py index 5c7a90c..57d53a1 100644 --- a/backend-compliance/compliance/services/doc_checks/llm_verify.py +++ b/backend-compliance/compliance/services/doc_checks/llm_verify.py @@ -24,32 +24,25 @@ async def verify_failed_checks( failed_checks: list[dict], doc_title: str, ) -> dict[str, dict]: - """Verify regex FAIL results using LLM. + """Verify regex FAIL results using LLM — single batched call. - For each failed check, asks the LLM a binary YES/NO question. + Sends ALL failed checks in one LLM prompt instead of one call per check. Returns a dict mapping check_id -> {"overturned": bool, "evidence": str}. - - Only checks with a "hint" field are verified (hints contain the - natural-language question the LLM can answer). """ results: dict[str, dict] = {} - if not failed_checks: + checks_with_hints = [c for c in failed_checks if c.get("hint")] + if not checks_with_hints: return results # Truncate text to fit context window text_excerpt = text[:8000] - for check in failed_checks: - check_id = check.get("id", "") - label = check.get("label", "") - hint = check.get("hint", "") - - if not hint: - continue - - try: - answer = await _ask_llm(text_excerpt, label, hint, doc_title) + try: + batch_results = await _ask_llm_batch( + text_excerpt, checks_with_hints, doc_title, + ) + for check_id, answer in batch_results.items(): overturned = answer.get("found", False) results[check_id] = { "overturned": overturned, @@ -58,71 +51,88 @@ async def verify_failed_checks( if overturned: logger.info( "LLM overturned regex FAIL for '%s' in '%s': %s", - label, doc_title, answer.get("evidence", "")[:80], + check_id, doc_title, answer.get("evidence", "")[:80], ) - except Exception as e: - logger.warning("LLM verify failed for '%s': %s", label, e) + except Exception as e: + logger.warning("LLM batch verify failed for '%s': %s", doc_title, e) return results -async def _ask_llm( - text: str, check_label: str, hint: str, doc_title: str, -) -> dict: - """Ask the LLM a binary verification question.""" - prompt = f"""/no_think -Pruefe ob der folgende Dokumenttext die Anforderung erfuellt. +async def _ask_llm_batch( + text: str, checks: list[dict], doc_title: str, +) -> dict[str, dict]: + """Ask the LLM to verify ALL failed checks in a single call.""" + checklist_lines = [] + for i, c in enumerate(checks, 1): + checklist_lines.append( + f'{i}. ID="{c["id"]}" | {c["label"]} | {c.get("hint", "")[:120]}' + ) + checklist_str = "\n".join(checklist_lines) -ANFORDERUNG: {check_label} -DETAILS: {hint} + prompt = f"""/no_think +Pruefe ob der Dokumenttext die folgenden Anforderungen erfuellt. DOKUMENT: "{doc_title}" +ANFORDERUNGEN: +{checklist_str} + TEXT: {text} -Antworte NUR mit einem JSON-Objekt (keine Erklaerung): -{{"found": true/false, "evidence": "Zitat aus dem Text das die Anforderung belegt (max 100 Zeichen), oder leer wenn nicht gefunden"}} +Antworte NUR mit einem JSON-Array (keine Erklaerung). Fuer jede Anforderung: +[{{"id": "check-id", "found": true/false, "evidence": "Kurzes Zitat (max 80 Zeichen) oder leer"}}] """ - async with httpx.AsyncClient(timeout=TIMEOUT) as client: + async with httpx.AsyncClient(timeout=90.0) as client: resp = await client.post( f"{OLLAMA_URL}/api/generate", json={ "model": OLLAMA_MODEL, "prompt": prompt, "stream": False, - "options": {"temperature": 0.0, "num_predict": 200}, + "options": {"temperature": 0.0, "num_predict": 2000}, }, ) resp.raise_for_status() raw = resp.json().get("response", "") - return _parse_llm_response(raw) + return _parse_batch_response(raw, checks) -def _parse_llm_response(raw: str) -> dict: - """Parse LLM JSON response with fallback extraction.""" +def _parse_batch_response(raw: str, checks: list[dict]) -> dict[str, dict]: + """Parse batch LLM JSON array response.""" import json import re - # Try direct JSON parse + results: dict[str, dict] = {} raw = raw.strip() - # Extract JSON from markdown code blocks - m = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", raw, re.DOTALL) + + # Extract JSON array from markdown code blocks + m = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", raw, re.DOTALL) if m: raw = m.group(1) - # Or just find the JSON object - m = re.search(r"\{[^}]*\"found\"[^}]*\}", raw, re.DOTALL) - if m: - raw = m.group(0) + else: + m = re.search(r"\[.*\]", raw, re.DOTALL) + if m: + raw = m.group(0) try: - data = json.loads(raw) - return { - "found": bool(data.get("found", False)), - "evidence": str(data.get("evidence", ""))[:150], - } + items = json.loads(raw) + if isinstance(items, list): + for item in items: + cid = item.get("id", "") + if cid: + results[cid] = { + "found": bool(item.get("found", False)), + "evidence": str(item.get("evidence", ""))[:150], + } except (json.JSONDecodeError, ValueError): - # Fallback: look for "found": true/false - found = '"found": true' in raw.lower() or '"found":true' in raw.lower() - return {"found": found, "evidence": ""} + # Fallback: extract individual JSON objects + for m in re.finditer(r'\{[^}]*"id"\s*:\s*"([^"]+)"[^}]*"found"\s*:\s*(true|false)[^}]*\}', raw, re.DOTALL): + cid = m.group(1) + found = m.group(2) == "true" + results[cid] = {"found": found, "evidence": ""} + + logger.info("LLM batch: %d/%d checks parsed", len(results), len(checks)) + return results