bfdaf63ba9
A previous `git pull --rebase origin main` dropped 177 local commits,
losing 3400+ files across admin-v2, backend, studio-v2, website,
klausur-service, and many other services. The partial restore attempt
(660295e2) only recovered some files.
This commit restores all missing files from pre-rebase ref 98933f5e
while preserving post-rebase additions (night-scheduler, night-mode UI,
NightModeWidget dashboard integration).
Restored features include:
- AI Module Sidebar (FAB), OCR Labeling, OCR Compare
- GPU Dashboard, RAG Pipeline, Magic Help
- Klausur-Korrektur (8 files), Abitur-Archiv (5+ files)
- Companion, Zeugnisse-Crawler, Screen Flow
- Full backend, studio-v2, website, klausur-service
- All compliance SDKs, agent-core, voice-service
- CI/CD configs, documentation, scripts
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
456 lines
16 KiB
Python
456 lines
16 KiB
Python
"""
|
|
LLM Compare Test API - Test Runner fuer LLM Provider Vergleich
|
|
Endpoint: /api/admin/llm-tests
|
|
"""
|
|
|
|
from fastapi import APIRouter
|
|
from pydantic import BaseModel
|
|
from typing import List, Optional, Literal
|
|
import httpx
|
|
import asyncio
|
|
import time
|
|
import os
|
|
|
|
router = APIRouter(prefix="/api/admin/llm-tests", tags=["LLM Tests"])
|
|
|
|
# ==============================================
|
|
# Models
|
|
# ==============================================
|
|
|
|
class TestResult(BaseModel):
|
|
name: str
|
|
description: str
|
|
expected: str
|
|
actual: str
|
|
status: Literal["passed", "failed", "pending", "skipped"]
|
|
duration_ms: float
|
|
error_message: Optional[str] = None
|
|
|
|
|
|
class TestCategoryResult(BaseModel):
|
|
category: str
|
|
display_name: str
|
|
description: str
|
|
tests: List[TestResult]
|
|
passed: int
|
|
failed: int
|
|
total: int
|
|
|
|
|
|
class FullTestResults(BaseModel):
|
|
categories: List[TestCategoryResult]
|
|
total_passed: int
|
|
total_failed: int
|
|
total_tests: int
|
|
duration_ms: float
|
|
|
|
|
|
# ==============================================
|
|
# Configuration
|
|
# ==============================================
|
|
|
|
BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8000")
|
|
LLM_GATEWAY_ENABLED = os.getenv("LLM_GATEWAY_ENABLED", "false").lower() == "true"
|
|
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
|
|
ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")
|
|
|
|
|
|
# ==============================================
|
|
# Test Implementations
|
|
# ==============================================
|
|
|
|
async def test_llm_gateway_health() -> TestResult:
|
|
"""Test LLM Gateway Health Endpoint"""
|
|
start = time.time()
|
|
|
|
if not LLM_GATEWAY_ENABLED:
|
|
return TestResult(
|
|
name="LLM Gateway Health",
|
|
description="Prueft ob das LLM Gateway aktiviert und erreichbar ist",
|
|
expected="LLM Gateway aktiv",
|
|
actual="LLM_GATEWAY_ENABLED=false",
|
|
status="skipped",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message="LLM Gateway nicht aktiviert"
|
|
)
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
|
response = await client.get(f"{BACKEND_URL}/llm/health")
|
|
duration = (time.time() - start) * 1000
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
providers = data.get("providers", [])
|
|
return TestResult(
|
|
name="LLM Gateway Health",
|
|
description="Prueft ob das LLM Gateway aktiviert und erreichbar ist",
|
|
expected="LLM Gateway aktiv",
|
|
actual=f"Aktiv, {len(providers)} Provider",
|
|
status="passed",
|
|
duration_ms=duration
|
|
)
|
|
else:
|
|
return TestResult(
|
|
name="LLM Gateway Health",
|
|
description="Prueft ob das LLM Gateway aktiviert und erreichbar ist",
|
|
expected="LLM Gateway aktiv",
|
|
actual=f"HTTP {response.status_code}",
|
|
status="failed",
|
|
duration_ms=duration,
|
|
error_message="Gateway nicht erreichbar"
|
|
)
|
|
except Exception as e:
|
|
return TestResult(
|
|
name="LLM Gateway Health",
|
|
description="Prueft ob das LLM Gateway aktiviert und erreichbar ist",
|
|
expected="LLM Gateway aktiv",
|
|
actual=f"Fehler: {str(e)}",
|
|
status="failed",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message=str(e)
|
|
)
|
|
|
|
|
|
async def test_openai_api() -> TestResult:
|
|
"""Test OpenAI API Connection"""
|
|
start = time.time()
|
|
|
|
if not OPENAI_API_KEY:
|
|
return TestResult(
|
|
name="OpenAI API Verbindung",
|
|
description="Prueft ob die OpenAI API konfiguriert und erreichbar ist",
|
|
expected="API Key konfiguriert",
|
|
actual="OPENAI_API_KEY nicht gesetzt",
|
|
status="skipped",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message="API Key fehlt"
|
|
)
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
|
response = await client.get(
|
|
"https://api.openai.com/v1/models",
|
|
headers={"Authorization": f"Bearer {OPENAI_API_KEY}"}
|
|
)
|
|
duration = (time.time() - start) * 1000
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
models = [m["id"] for m in data.get("data", [])[:5]]
|
|
return TestResult(
|
|
name="OpenAI API Verbindung",
|
|
description="Prueft ob die OpenAI API konfiguriert und erreichbar ist",
|
|
expected="API erreichbar mit Modellen",
|
|
actual=f"Verfuegbar: {', '.join(models[:3])}...",
|
|
status="passed",
|
|
duration_ms=duration
|
|
)
|
|
else:
|
|
return TestResult(
|
|
name="OpenAI API Verbindung",
|
|
description="Prueft ob die OpenAI API konfiguriert und erreichbar ist",
|
|
expected="API erreichbar mit Modellen",
|
|
actual=f"HTTP {response.status_code}",
|
|
status="failed",
|
|
duration_ms=duration,
|
|
error_message="API-Authentifizierung fehlgeschlagen"
|
|
)
|
|
except Exception as e:
|
|
return TestResult(
|
|
name="OpenAI API Verbindung",
|
|
description="Prueft ob die OpenAI API konfiguriert und erreichbar ist",
|
|
expected="API erreichbar mit Modellen",
|
|
actual=f"Fehler: {str(e)}",
|
|
status="failed",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message=str(e)
|
|
)
|
|
|
|
|
|
async def test_anthropic_api() -> TestResult:
|
|
"""Test Anthropic API Connection"""
|
|
start = time.time()
|
|
|
|
if not ANTHROPIC_API_KEY:
|
|
return TestResult(
|
|
name="Anthropic API Verbindung",
|
|
description="Prueft ob die Anthropic (Claude) API konfiguriert ist",
|
|
expected="API Key konfiguriert",
|
|
actual="ANTHROPIC_API_KEY nicht gesetzt",
|
|
status="skipped",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message="API Key fehlt"
|
|
)
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
|
# Anthropic doesn't have a models endpoint, so we do a minimal completion
|
|
response = await client.post(
|
|
"https://api.anthropic.com/v1/messages",
|
|
headers={
|
|
"x-api-key": ANTHROPIC_API_KEY,
|
|
"anthropic-version": "2023-06-01",
|
|
"content-type": "application/json"
|
|
},
|
|
json={
|
|
"model": "claude-3-haiku-20240307",
|
|
"max_tokens": 10,
|
|
"messages": [{"role": "user", "content": "Hi"}]
|
|
}
|
|
)
|
|
duration = (time.time() - start) * 1000
|
|
|
|
if response.status_code == 200:
|
|
return TestResult(
|
|
name="Anthropic API Verbindung",
|
|
description="Prueft ob die Anthropic (Claude) API konfiguriert ist",
|
|
expected="API erreichbar",
|
|
actual="Claude API verfuegbar",
|
|
status="passed",
|
|
duration_ms=duration
|
|
)
|
|
elif response.status_code == 401:
|
|
return TestResult(
|
|
name="Anthropic API Verbindung",
|
|
description="Prueft ob die Anthropic (Claude) API konfiguriert ist",
|
|
expected="API erreichbar",
|
|
actual="Ungueltige Credentials",
|
|
status="failed",
|
|
duration_ms=duration,
|
|
error_message="API Key ungueltig"
|
|
)
|
|
else:
|
|
return TestResult(
|
|
name="Anthropic API Verbindung",
|
|
description="Prueft ob die Anthropic (Claude) API konfiguriert ist",
|
|
expected="API erreichbar",
|
|
actual=f"HTTP {response.status_code}",
|
|
status="failed",
|
|
duration_ms=duration,
|
|
error_message=f"Unerwarteter Status: {response.status_code}"
|
|
)
|
|
except Exception as e:
|
|
return TestResult(
|
|
name="Anthropic API Verbindung",
|
|
description="Prueft ob die Anthropic (Claude) API konfiguriert ist",
|
|
expected="API erreichbar",
|
|
actual=f"Fehler: {str(e)}",
|
|
status="failed",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message=str(e)
|
|
)
|
|
|
|
|
|
async def test_local_llm() -> TestResult:
|
|
"""Test Local LLM (Ollama) Connection"""
|
|
start = time.time()
|
|
ollama_url = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=5.0) as client:
|
|
response = await client.get(f"{ollama_url}/api/tags")
|
|
duration = (time.time() - start) * 1000
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
models = [m["name"] for m in data.get("models", [])]
|
|
if models:
|
|
return TestResult(
|
|
name="Lokales LLM (Ollama)",
|
|
description="Prueft ob ein lokales LLM via Ollama verfuegbar ist",
|
|
expected="Ollama mit Modellen",
|
|
actual=f"Modelle: {', '.join(models[:3])}",
|
|
status="passed",
|
|
duration_ms=duration
|
|
)
|
|
else:
|
|
return TestResult(
|
|
name="Lokales LLM (Ollama)",
|
|
description="Prueft ob ein lokales LLM via Ollama verfuegbar ist",
|
|
expected="Ollama mit Modellen",
|
|
actual="Ollama aktiv, keine Modelle",
|
|
status="passed",
|
|
duration_ms=duration
|
|
)
|
|
else:
|
|
return TestResult(
|
|
name="Lokales LLM (Ollama)",
|
|
description="Prueft ob ein lokales LLM via Ollama verfuegbar ist",
|
|
expected="Ollama mit Modellen",
|
|
actual=f"HTTP {response.status_code}",
|
|
status="skipped",
|
|
duration_ms=duration,
|
|
error_message="Ollama nicht erreichbar"
|
|
)
|
|
except Exception as e:
|
|
return TestResult(
|
|
name="Lokales LLM (Ollama)",
|
|
description="Prueft ob ein lokales LLM via Ollama verfuegbar ist",
|
|
expected="Ollama mit Modellen",
|
|
actual="Nicht verfuegbar",
|
|
status="skipped",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message=str(e)
|
|
)
|
|
|
|
|
|
async def test_playbooks_api() -> TestResult:
|
|
"""Test LLM Playbooks API"""
|
|
start = time.time()
|
|
|
|
if not LLM_GATEWAY_ENABLED:
|
|
return TestResult(
|
|
name="LLM Playbooks API",
|
|
description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist",
|
|
expected="Playbooks API verfuegbar",
|
|
actual="LLM Gateway deaktiviert",
|
|
status="skipped",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message="LLM_GATEWAY_ENABLED=false"
|
|
)
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=10.0) as client:
|
|
response = await client.get(f"{BACKEND_URL}/llm/playbooks")
|
|
duration = (time.time() - start) * 1000
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
count = len(data) if isinstance(data, list) else data.get("total", 0)
|
|
return TestResult(
|
|
name="LLM Playbooks API",
|
|
description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist",
|
|
expected="Playbooks API verfuegbar",
|
|
actual=f"{count} Playbooks verfuegbar",
|
|
status="passed",
|
|
duration_ms=duration
|
|
)
|
|
else:
|
|
return TestResult(
|
|
name="LLM Playbooks API",
|
|
description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist",
|
|
expected="Playbooks API verfuegbar",
|
|
actual=f"HTTP {response.status_code}",
|
|
status="failed",
|
|
duration_ms=duration,
|
|
error_message="API nicht erreichbar"
|
|
)
|
|
except Exception as e:
|
|
return TestResult(
|
|
name="LLM Playbooks API",
|
|
description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist",
|
|
expected="Playbooks API verfuegbar",
|
|
actual=f"Fehler: {str(e)}",
|
|
status="failed",
|
|
duration_ms=(time.time() - start) * 1000,
|
|
error_message=str(e)
|
|
)
|
|
|
|
|
|
# ==============================================
|
|
# Category Runners
|
|
# ==============================================
|
|
|
|
async def run_gateway_tests() -> TestCategoryResult:
|
|
"""Run LLM Gateway tests"""
|
|
tests = await asyncio.gather(
|
|
test_llm_gateway_health(),
|
|
test_playbooks_api(),
|
|
)
|
|
|
|
passed = sum(1 for t in tests if t.status == "passed")
|
|
failed = sum(1 for t in tests if t.status == "failed")
|
|
|
|
return TestCategoryResult(
|
|
category="gateway",
|
|
display_name="LLM Gateway",
|
|
description="Tests fuer das zentrale LLM Gateway",
|
|
tests=list(tests),
|
|
passed=passed,
|
|
failed=failed,
|
|
total=len(tests)
|
|
)
|
|
|
|
|
|
async def run_provider_tests() -> TestCategoryResult:
|
|
"""Run LLM Provider tests"""
|
|
tests = await asyncio.gather(
|
|
test_openai_api(),
|
|
test_anthropic_api(),
|
|
test_local_llm(),
|
|
)
|
|
|
|
passed = sum(1 for t in tests if t.status == "passed")
|
|
failed = sum(1 for t in tests if t.status == "failed")
|
|
|
|
return TestCategoryResult(
|
|
category="providers",
|
|
display_name="LLM Provider",
|
|
description="Tests fuer externe und lokale LLM Provider",
|
|
tests=list(tests),
|
|
passed=passed,
|
|
failed=failed,
|
|
total=len(tests)
|
|
)
|
|
|
|
|
|
# ==============================================
|
|
# API Endpoints
|
|
# ==============================================
|
|
|
|
@router.post("/{category}", response_model=TestCategoryResult)
|
|
async def run_category_tests(category: str):
|
|
"""Run tests for a specific category"""
|
|
runners = {
|
|
"gateway": run_gateway_tests,
|
|
"providers": run_provider_tests,
|
|
}
|
|
|
|
if category not in runners:
|
|
return TestCategoryResult(
|
|
category=category,
|
|
display_name=f"Unbekannt: {category}",
|
|
description="Kategorie nicht gefunden",
|
|
tests=[],
|
|
passed=0,
|
|
failed=0,
|
|
total=0
|
|
)
|
|
|
|
return await runners[category]()
|
|
|
|
|
|
@router.post("/run-all", response_model=FullTestResults)
|
|
async def run_all_tests():
|
|
"""Run all LLM tests"""
|
|
start = time.time()
|
|
|
|
categories = await asyncio.gather(
|
|
run_gateway_tests(),
|
|
run_provider_tests(),
|
|
)
|
|
|
|
total_passed = sum(c.passed for c in categories)
|
|
total_failed = sum(c.failed for c in categories)
|
|
total_tests = sum(c.total for c in categories)
|
|
|
|
return FullTestResults(
|
|
categories=list(categories),
|
|
total_passed=total_passed,
|
|
total_failed=total_failed,
|
|
total_tests=total_tests,
|
|
duration_ms=(time.time() - start) * 1000
|
|
)
|
|
|
|
|
|
@router.get("/categories")
|
|
async def get_categories():
|
|
"""Get available test categories"""
|
|
return {
|
|
"categories": [
|
|
{"id": "gateway", "name": "LLM Gateway", "description": "Gateway Health & Playbooks"},
|
|
{"id": "providers", "name": "LLM Provider", "description": "OpenAI, Anthropic, Ollama"},
|
|
]
|
|
}
|