""" LLM Compare Test API - Test Runner fuer LLM Provider Vergleich Endpoint: /api/admin/llm-tests """ from fastapi import APIRouter from pydantic import BaseModel from typing import List, Optional, Literal import httpx import asyncio import time import os router = APIRouter(prefix="/api/admin/llm-tests", tags=["LLM Tests"]) # ============================================== # Models # ============================================== class TestResult(BaseModel): name: str description: str expected: str actual: str status: Literal["passed", "failed", "pending", "skipped"] duration_ms: float error_message: Optional[str] = None class TestCategoryResult(BaseModel): category: str display_name: str description: str tests: List[TestResult] passed: int failed: int total: int class FullTestResults(BaseModel): categories: List[TestCategoryResult] total_passed: int total_failed: int total_tests: int duration_ms: float # ============================================== # Configuration # ============================================== BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8000") LLM_GATEWAY_ENABLED = os.getenv("LLM_GATEWAY_ENABLED", "false").lower() == "true" OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "") # ============================================== # Test Implementations # ============================================== async def test_llm_gateway_health() -> TestResult: """Test LLM Gateway Health Endpoint""" start = time.time() if not LLM_GATEWAY_ENABLED: return TestResult( name="LLM Gateway Health", description="Prueft ob das LLM Gateway aktiviert und erreichbar ist", expected="LLM Gateway aktiv", actual="LLM_GATEWAY_ENABLED=false", status="skipped", duration_ms=(time.time() - start) * 1000, error_message="LLM Gateway nicht aktiviert" ) try: async with httpx.AsyncClient(timeout=10.0) as client: response = await client.get(f"{BACKEND_URL}/llm/health") duration = (time.time() - start) * 1000 if response.status_code == 200: data = response.json() providers = data.get("providers", []) return TestResult( name="LLM Gateway Health", description="Prueft ob das LLM Gateway aktiviert und erreichbar ist", expected="LLM Gateway aktiv", actual=f"Aktiv, {len(providers)} Provider", status="passed", duration_ms=duration ) else: return TestResult( name="LLM Gateway Health", description="Prueft ob das LLM Gateway aktiviert und erreichbar ist", expected="LLM Gateway aktiv", actual=f"HTTP {response.status_code}", status="failed", duration_ms=duration, error_message="Gateway nicht erreichbar" ) except Exception as e: return TestResult( name="LLM Gateway Health", description="Prueft ob das LLM Gateway aktiviert und erreichbar ist", expected="LLM Gateway aktiv", actual=f"Fehler: {str(e)}", status="failed", duration_ms=(time.time() - start) * 1000, error_message=str(e) ) async def test_openai_api() -> TestResult: """Test OpenAI API Connection""" start = time.time() if not OPENAI_API_KEY: return TestResult( name="OpenAI API Verbindung", description="Prueft ob die OpenAI API konfiguriert und erreichbar ist", expected="API Key konfiguriert", actual="OPENAI_API_KEY nicht gesetzt", status="skipped", duration_ms=(time.time() - start) * 1000, error_message="API Key fehlt" ) try: async with httpx.AsyncClient(timeout=10.0) as client: response = await client.get( "https://api.openai.com/v1/models", headers={"Authorization": f"Bearer {OPENAI_API_KEY}"} ) duration = (time.time() - start) * 1000 if response.status_code == 200: data = response.json() models = [m["id"] for m in data.get("data", [])[:5]] return TestResult( name="OpenAI API Verbindung", description="Prueft ob die OpenAI API konfiguriert und erreichbar ist", expected="API erreichbar mit Modellen", actual=f"Verfuegbar: {', '.join(models[:3])}...", status="passed", duration_ms=duration ) else: return TestResult( name="OpenAI API Verbindung", description="Prueft ob die OpenAI API konfiguriert und erreichbar ist", expected="API erreichbar mit Modellen", actual=f"HTTP {response.status_code}", status="failed", duration_ms=duration, error_message="API-Authentifizierung fehlgeschlagen" ) except Exception as e: return TestResult( name="OpenAI API Verbindung", description="Prueft ob die OpenAI API konfiguriert und erreichbar ist", expected="API erreichbar mit Modellen", actual=f"Fehler: {str(e)}", status="failed", duration_ms=(time.time() - start) * 1000, error_message=str(e) ) async def test_anthropic_api() -> TestResult: """Test Anthropic API Connection""" start = time.time() if not ANTHROPIC_API_KEY: return TestResult( name="Anthropic API Verbindung", description="Prueft ob die Anthropic (Claude) API konfiguriert ist", expected="API Key konfiguriert", actual="ANTHROPIC_API_KEY nicht gesetzt", status="skipped", duration_ms=(time.time() - start) * 1000, error_message="API Key fehlt" ) try: async with httpx.AsyncClient(timeout=10.0) as client: # Anthropic doesn't have a models endpoint, so we do a minimal completion response = await client.post( "https://api.anthropic.com/v1/messages", headers={ "x-api-key": ANTHROPIC_API_KEY, "anthropic-version": "2023-06-01", "content-type": "application/json" }, json={ "model": "claude-3-haiku-20240307", "max_tokens": 10, "messages": [{"role": "user", "content": "Hi"}] } ) duration = (time.time() - start) * 1000 if response.status_code == 200: return TestResult( name="Anthropic API Verbindung", description="Prueft ob die Anthropic (Claude) API konfiguriert ist", expected="API erreichbar", actual="Claude API verfuegbar", status="passed", duration_ms=duration ) elif response.status_code == 401: return TestResult( name="Anthropic API Verbindung", description="Prueft ob die Anthropic (Claude) API konfiguriert ist", expected="API erreichbar", actual="Ungueltige Credentials", status="failed", duration_ms=duration, error_message="API Key ungueltig" ) else: return TestResult( name="Anthropic API Verbindung", description="Prueft ob die Anthropic (Claude) API konfiguriert ist", expected="API erreichbar", actual=f"HTTP {response.status_code}", status="failed", duration_ms=duration, error_message=f"Unerwarteter Status: {response.status_code}" ) except Exception as e: return TestResult( name="Anthropic API Verbindung", description="Prueft ob die Anthropic (Claude) API konfiguriert ist", expected="API erreichbar", actual=f"Fehler: {str(e)}", status="failed", duration_ms=(time.time() - start) * 1000, error_message=str(e) ) async def test_local_llm() -> TestResult: """Test Local LLM (Ollama) Connection""" start = time.time() ollama_url = os.getenv("OLLAMA_URL", "http://localhost:11434") try: async with httpx.AsyncClient(timeout=5.0) as client: response = await client.get(f"{ollama_url}/api/tags") duration = (time.time() - start) * 1000 if response.status_code == 200: data = response.json() models = [m["name"] for m in data.get("models", [])] if models: return TestResult( name="Lokales LLM (Ollama)", description="Prueft ob ein lokales LLM via Ollama verfuegbar ist", expected="Ollama mit Modellen", actual=f"Modelle: {', '.join(models[:3])}", status="passed", duration_ms=duration ) else: return TestResult( name="Lokales LLM (Ollama)", description="Prueft ob ein lokales LLM via Ollama verfuegbar ist", expected="Ollama mit Modellen", actual="Ollama aktiv, keine Modelle", status="passed", duration_ms=duration ) else: return TestResult( name="Lokales LLM (Ollama)", description="Prueft ob ein lokales LLM via Ollama verfuegbar ist", expected="Ollama mit Modellen", actual=f"HTTP {response.status_code}", status="skipped", duration_ms=duration, error_message="Ollama nicht erreichbar" ) except Exception as e: return TestResult( name="Lokales LLM (Ollama)", description="Prueft ob ein lokales LLM via Ollama verfuegbar ist", expected="Ollama mit Modellen", actual="Nicht verfuegbar", status="skipped", duration_ms=(time.time() - start) * 1000, error_message=str(e) ) async def test_playbooks_api() -> TestResult: """Test LLM Playbooks API""" start = time.time() if not LLM_GATEWAY_ENABLED: return TestResult( name="LLM Playbooks API", description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist", expected="Playbooks API verfuegbar", actual="LLM Gateway deaktiviert", status="skipped", duration_ms=(time.time() - start) * 1000, error_message="LLM_GATEWAY_ENABLED=false" ) try: async with httpx.AsyncClient(timeout=10.0) as client: response = await client.get(f"{BACKEND_URL}/llm/playbooks") duration = (time.time() - start) * 1000 if response.status_code == 200: data = response.json() count = len(data) if isinstance(data, list) else data.get("total", 0) return TestResult( name="LLM Playbooks API", description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist", expected="Playbooks API verfuegbar", actual=f"{count} Playbooks verfuegbar", status="passed", duration_ms=duration ) else: return TestResult( name="LLM Playbooks API", description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist", expected="Playbooks API verfuegbar", actual=f"HTTP {response.status_code}", status="failed", duration_ms=duration, error_message="API nicht erreichbar" ) except Exception as e: return TestResult( name="LLM Playbooks API", description="Prueft ob die Playbooks-Verwaltung fuer vordefinierte Prompts verfuegbar ist", expected="Playbooks API verfuegbar", actual=f"Fehler: {str(e)}", status="failed", duration_ms=(time.time() - start) * 1000, error_message=str(e) ) # ============================================== # Category Runners # ============================================== async def run_gateway_tests() -> TestCategoryResult: """Run LLM Gateway tests""" tests = await asyncio.gather( test_llm_gateway_health(), test_playbooks_api(), ) passed = sum(1 for t in tests if t.status == "passed") failed = sum(1 for t in tests if t.status == "failed") return TestCategoryResult( category="gateway", display_name="LLM Gateway", description="Tests fuer das zentrale LLM Gateway", tests=list(tests), passed=passed, failed=failed, total=len(tests) ) async def run_provider_tests() -> TestCategoryResult: """Run LLM Provider tests""" tests = await asyncio.gather( test_openai_api(), test_anthropic_api(), test_local_llm(), ) passed = sum(1 for t in tests if t.status == "passed") failed = sum(1 for t in tests if t.status == "failed") return TestCategoryResult( category="providers", display_name="LLM Provider", description="Tests fuer externe und lokale LLM Provider", tests=list(tests), passed=passed, failed=failed, total=len(tests) ) # ============================================== # API Endpoints # ============================================== @router.post("/{category}", response_model=TestCategoryResult) async def run_category_tests(category: str): """Run tests for a specific category""" runners = { "gateway": run_gateway_tests, "providers": run_provider_tests, } if category not in runners: return TestCategoryResult( category=category, display_name=f"Unbekannt: {category}", description="Kategorie nicht gefunden", tests=[], passed=0, failed=0, total=0 ) return await runners[category]() @router.post("/run-all", response_model=FullTestResults) async def run_all_tests(): """Run all LLM tests""" start = time.time() categories = await asyncio.gather( run_gateway_tests(), run_provider_tests(), ) total_passed = sum(c.passed for c in categories) total_failed = sum(c.failed for c in categories) total_tests = sum(c.total for c in categories) return FullTestResults( categories=list(categories), total_passed=total_passed, total_failed=total_failed, total_tests=total_tests, duration_ms=(time.time() - start) * 1000 ) @router.get("/categories") async def get_categories(): """Get available test categories""" return { "categories": [ {"id": "gateway", "name": "LLM Gateway", "description": "Gateway Health & Playbooks"}, {"id": "providers", "name": "LLM Provider", "description": "OpenAI, Anthropic, Ollama"}, ] }