Initial commit: breakpilot-lehrer - Lehrer KI Platform
Services: Admin-Lehrer, Backend-Lehrer, Studio v2, Website, Klausur-Service, School-Service, Voice-Service, Geo-Service, BreakPilot Drive, Agent-Core Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Boenisch
@@ -0,0 +1,164 @@
|
||||
import type { SystemInfoConfig } from './types'
|
||||
|
||||
export const llmCompareConfig: SystemInfoConfig = {
|
||||
title: 'LLM Vergleich System-Info',
|
||||
description: 'Vergleich und Benchmarking verschiedener KI-Provider und Modelle.',
|
||||
version: '1.0',
|
||||
architecture: {
|
||||
layers: [
|
||||
{ title: 'Vergleichs-UI', components: ['Provider-Auswahl', 'Prompt-Editor', 'Ergebnis-Vergleich'], color: '#3b82f6' },
|
||||
{ title: 'Provider Adapters', components: ['OpenAI', 'Anthropic', 'Google', 'Local'], color: '#8b5cf6' },
|
||||
{ title: 'Evaluation Engine', components: ['Latenz-Messung', 'Qualitaets-Scoring', 'Cost Calculator'], color: '#10b981' },
|
||||
{ title: 'Logging', components: ['Request Logs', 'Token Tracking', 'Error Logs'], color: '#f59e0b' },
|
||||
],
|
||||
},
|
||||
features: [
|
||||
{ name: 'Multi-Provider Vergleich', status: 'active', description: 'Parallele Anfragen an mehrere LLMs' },
|
||||
{ name: 'Latenz-Tracking', status: 'active', description: 'Echtzeit-Performance-Messung' },
|
||||
{ name: 'Kosten-Kalkulation', status: 'active', description: 'Token-basierte Kostenberechnung' },
|
||||
{ name: 'Qualitaets-Bewertung', status: 'planned', description: 'Automatisches Scoring der Antworten' },
|
||||
{ name: 'A/B Testing', status: 'planned', description: 'Statistische Signifikanz-Tests' },
|
||||
],
|
||||
roadmap: [
|
||||
{ phase: 'Phase 1: Provider (Q1)', priority: 'high', items: ['Mistral Integration', 'Llama 3 Integration', 'Gemini Pro Integration', 'Rate Limiting'] },
|
||||
{ phase: 'Phase 2: Evaluation (Q2)', priority: 'high', items: ['Automatisches Scoring', 'Benchmark-Suite', 'Domain-spezifische Tests', 'Halluzinations-Erkennung'] },
|
||||
{ phase: 'Phase 3: Optimierung (Q3)', priority: 'medium', items: ['Prompt-Optimierung', 'Modell-Routing', 'Fallback-Strategien', 'Caching'] },
|
||||
],
|
||||
technicalDetails: [
|
||||
{ component: 'OpenAI', technology: 'GPT-4o / o1', description: 'Haupt-Provider' },
|
||||
{ component: 'Anthropic', technology: 'Claude 3.5', description: 'Alternative' },
|
||||
{ component: 'Google', technology: 'Gemini 2.0', description: 'Multimodal' },
|
||||
{ component: 'Local', technology: 'Ollama', description: 'Self-hosted' },
|
||||
],
|
||||
auditInfo: [
|
||||
{
|
||||
category: 'Provider-Status',
|
||||
items: [
|
||||
{ label: 'OpenAI', value: 'Aktiv', status: 'ok' },
|
||||
{ label: 'Anthropic', value: 'Aktiv', status: 'ok' },
|
||||
{ label: 'Google Gemini', value: 'Aktiv', status: 'ok' },
|
||||
{ label: 'Ollama (Local)', value: 'Verfuegbar', status: 'ok' },
|
||||
],
|
||||
},
|
||||
{
|
||||
category: 'Kosten & Limits',
|
||||
items: [
|
||||
{ label: 'Monatliches Budget', value: 'Konfigurierbar', status: 'ok' },
|
||||
{ label: 'Rate Limiting', value: 'Pro Provider', status: 'ok' },
|
||||
{ label: 'Token Tracking', value: 'Aktiviert', status: 'ok' },
|
||||
{ label: 'Cost Alerts', value: 'E-Mail', status: 'ok' },
|
||||
],
|
||||
},
|
||||
{
|
||||
category: 'Datenschutz',
|
||||
items: [
|
||||
{ label: 'Prompt-Logging', value: 'Optional', status: 'ok' },
|
||||
{ label: 'PII Detection', value: 'Geplant', status: 'warning' },
|
||||
{ label: 'Data Residency', value: 'EU verfuegbar', status: 'ok' },
|
||||
{ label: 'Audit-Log', value: 'Aktiviert', status: 'ok' },
|
||||
],
|
||||
},
|
||||
],
|
||||
fullDocumentation: `
|
||||
<h2>LLM Provider Vergleich & Benchmarking</h2>
|
||||
|
||||
<h3>1. Uebersicht</h3>
|
||||
<p>Das LLM-Vergleichsmodul ermoeglicht den direkten Vergleich verschiedener KI-Provider hinsichtlich Qualitaet, Latenz und Kosten. Es dient der Auswahl des optimalen Modells fuer spezifische Use Cases.</p>
|
||||
|
||||
<h3>2. Unterstuetzte Provider</h3>
|
||||
<table>
|
||||
<tr><th>Provider</th><th>Modelle</th><th>Staerken</th><th>Preisbereich</th></tr>
|
||||
<tr><td>OpenAI</td><td>GPT-4o, GPT-4o-mini, o1</td><td>Allrounder, Coding</td><td>$0.15-15/1M Token</td></tr>
|
||||
<tr><td>Anthropic</td><td>Claude 3.5 Sonnet/Haiku</td><td>Lange Kontexte, Safety</td><td>$0.25-15/1M Token</td></tr>
|
||||
<tr><td>Google</td><td>Gemini 2.0 Flash/Pro</td><td>Multimodal, Speed</td><td>$0.075-5/1M Token</td></tr>
|
||||
<tr><td>Ollama</td><td>Llama 3, Mistral, Phi</td><td>Lokal, Datenschutz</td><td>Nur Hardware-Kosten</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>3. Vergleichs-Architektur</h3>
|
||||
<pre>
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ Frontend UI │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐ │
|
||||
│ │ Prompt Input │ │ Provider Sel │ │ Results Comparison │ │
|
||||
│ └──────────────┘ └──────────────┘ └──────────────────────┘ │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
v
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ Backend Orchestrator │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ Parallel Request Handler | Response Aggregator │ │
|
||||
│ └──────────────────────────────────────────────────────────┘ │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
│ │ │ │
|
||||
v v v v
|
||||
┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐
|
||||
│ OpenAI │ │Anthropic│ │ Google │ │ Ollama │
|
||||
└─────────┘ └─────────┘ └─────────┘ └─────────┘
|
||||
</pre>
|
||||
|
||||
<h3>4. Metriken</h3>
|
||||
<table>
|
||||
<tr><th>Metrik</th><th>Beschreibung</th><th>Messung</th></tr>
|
||||
<tr><td>TTFB</td><td>Time to First Byte</td><td>Millisekunden</td></tr>
|
||||
<tr><td>Total Latency</td><td>Gesamtantwortzeit</td><td>Millisekunden</td></tr>
|
||||
<tr><td>Tokens/Sekunde</td><td>Generierungsgeschwindigkeit</td><td>Output Tokens/s</td></tr>
|
||||
<tr><td>Kosten</td><td>Gesamtkosten</td><td>USD</td></tr>
|
||||
<tr><td>Qualitaet</td><td>Manuelle/Auto Bewertung</td><td>1-5 Sterne</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>5. API Endpoints</h3>
|
||||
<table>
|
||||
<tr><th>Endpoint</th><th>Methode</th><th>Beschreibung</th></tr>
|
||||
<tr><td>/api/llm/compare</td><td>POST</td><td>Parallelen Vergleich starten</td></tr>
|
||||
<tr><td>/api/llm/providers</td><td>GET</td><td>Verfuegbare Provider listen</td></tr>
|
||||
<tr><td>/api/llm/stats</td><td>GET</td><td>Nutzungsstatistiken</td></tr>
|
||||
<tr><td>/api/llm/benchmark</td><td>POST</td><td>Benchmark-Suite ausfuehren</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>6. Benchmark-Suite</h3>
|
||||
<p>Vordefinierte Tests fuer verschiedene Use Cases:</p>
|
||||
<ul>
|
||||
<li><strong>Summarization:</strong> Textzusammenfassung verschiedener Laengen</li>
|
||||
<li><strong>QA:</strong> Frage-Antwort auf Dokumenten</li>
|
||||
<li><strong>Coding:</strong> Code-Generierung und -Erklaerung</li>
|
||||
<li><strong>Classification:</strong> Textkategorisierung</li>
|
||||
<li><strong>Translation:</strong> Mehrsprachige Uebersetzung</li>
|
||||
</ul>
|
||||
|
||||
<h3>7. Kostenmanagement</h3>
|
||||
<pre>
|
||||
Budgetkontrolle
|
||||
│
|
||||
├── Monatliches Limit pro Provider
|
||||
├── Echtzeit-Kostentracking
|
||||
├── Alerts bei 80%/90%/100%
|
||||
└── Auto-Fallback bei Limit
|
||||
</pre>
|
||||
|
||||
<h3>8. Datenschutz-Konfiguration</h3>
|
||||
<table>
|
||||
<tr><th>Einstellung</th><th>Optionen</th><th>Default</th></tr>
|
||||
<tr><td>Prompt-Logging</td><td>Ein/Aus/Anonymisiert</td><td>Anonymisiert</td></tr>
|
||||
<tr><td>Response-Speicherung</td><td>Ein/Aus/24h</td><td>24h</td></tr>
|
||||
<tr><td>Metriken-Retention</td><td>30/90/365 Tage</td><td>90 Tage</td></tr>
|
||||
<tr><td>PII-Filter</td><td>Ein/Aus</td><td>Ein (geplant)</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>9. Fehlerbehandlung</h3>
|
||||
<ul>
|
||||
<li><strong>Timeout:</strong> 30 Sekunden default, konfigurierbar</li>
|
||||
<li><strong>Rate Limit:</strong> Automatisches Retry mit Backoff</li>
|
||||
<li><strong>Provider Down:</strong> Skip und Warnung</li>
|
||||
<li><strong>API Error:</strong> Fehlerdetails in Response</li>
|
||||
</ul>
|
||||
|
||||
<h3>10. Best Practices</h3>
|
||||
<ol>
|
||||
<li>Immer mindestens 3 Provider fuer aussagekraeftigen Vergleich</li>
|
||||
<li>Benchmark-Suite fuer reproduzierbare Ergebnisse nutzen</li>
|
||||
<li>Kosten und Qualitaet gemeinsam bewerten</li>
|
||||
<li>Lokale Modelle fuer sensible Daten bevorzugen</li>
|
||||
</ol>
|
||||
`,
|
||||
}
|
||||
Reference in New Issue
Block a user