fix: Restore all files lost during destructive rebase
A previous `git pull --rebase origin main` dropped 177 local commits,
losing 3400+ files across admin-v2, backend, studio-v2, website,
klausur-service, and many other services. The partial restore attempt
(660295e2) only recovered some files.
This commit restores all missing files from pre-rebase ref 98933f5e
while preserving post-rebase additions (night-scheduler, night-mode UI,
NightModeWidget dashboard integration).
Restored features include:
- AI Module Sidebar (FAB), OCR Labeling, OCR Compare
- GPU Dashboard, RAG Pipeline, Magic Help
- Klausur-Korrektur (8 files), Abitur-Archiv (5+ files)
- Companion, Zeugnisse-Crawler, Screen Flow
- Full backend, studio-v2, website, klausur-service
- All compliance SDKs, agent-core, voice-service
- CI/CD configs, documentation, scripts
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
535
website/app/admin/uni-crawler/page.tsx
Normal file
535
website/app/admin/uni-crawler/page.tsx
Normal file
@@ -0,0 +1,535 @@
|
||||
'use client'
|
||||
|
||||
/**
|
||||
* University Crawler Control Panel
|
||||
*
|
||||
* Admin interface for managing the multi-phase university crawling system.
|
||||
* Allows starting/stopping the orchestrator, adding universities to the queue,
|
||||
* and monitoring crawl progress.
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useCallback } from 'react'
|
||||
import AdminLayout from '@/components/admin/AdminLayout'
|
||||
|
||||
// Types matching the Go backend
|
||||
interface CrawlQueueItem {
|
||||
id: string
|
||||
university_id: string
|
||||
university_name: string
|
||||
university_short: string
|
||||
queue_position: number | null
|
||||
priority: number
|
||||
current_phase: CrawlPhase
|
||||
discovery_completed: boolean
|
||||
discovery_completed_at?: string
|
||||
professors_completed: boolean
|
||||
professors_completed_at?: string
|
||||
all_staff_completed: boolean
|
||||
all_staff_completed_at?: string
|
||||
publications_completed: boolean
|
||||
publications_completed_at?: string
|
||||
discovery_count: number
|
||||
professors_count: number
|
||||
staff_count: number
|
||||
publications_count: number
|
||||
retry_count: number
|
||||
max_retries: number
|
||||
last_error?: string
|
||||
started_at?: string
|
||||
completed_at?: string
|
||||
progress_percent: number
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
|
||||
type CrawlPhase = 'pending' | 'discovery' | 'professors' | 'all_staff' | 'publications' | 'completed' | 'failed' | 'paused'
|
||||
|
||||
interface OrchestratorStatus {
|
||||
is_running: boolean
|
||||
current_university?: CrawlQueueItem
|
||||
current_phase: CrawlPhase
|
||||
queue_length: number
|
||||
completed_today: number
|
||||
total_processed: number
|
||||
last_activity?: string
|
||||
}
|
||||
|
||||
interface University {
|
||||
id: string
|
||||
name: string
|
||||
short_name?: string
|
||||
url: string
|
||||
state?: string
|
||||
uni_type?: string
|
||||
}
|
||||
|
||||
// Use local API proxy to avoid CORS issues and keep API keys server-side
|
||||
const API_BASE = '/api/admin/uni-crawler'
|
||||
|
||||
// Phase display configuration
|
||||
const phaseConfig: Record<CrawlPhase, { label: string; color: string; icon: string }> = {
|
||||
pending: { label: 'Wartend', color: 'bg-gray-100 text-gray-700', icon: 'clock' },
|
||||
discovery: { label: 'Discovery', color: 'bg-blue-100 text-blue-700', icon: 'search' },
|
||||
professors: { label: 'Professoren', color: 'bg-indigo-100 text-indigo-700', icon: 'user' },
|
||||
all_staff: { label: 'Alle Mitarbeiter', color: 'bg-purple-100 text-purple-700', icon: 'users' },
|
||||
publications: { label: 'Publikationen', color: 'bg-orange-100 text-orange-700', icon: 'book' },
|
||||
completed: { label: 'Abgeschlossen', color: 'bg-green-100 text-green-700', icon: 'check' },
|
||||
failed: { label: 'Fehlgeschlagen', color: 'bg-red-100 text-red-700', icon: 'x' },
|
||||
paused: { label: 'Pausiert', color: 'bg-yellow-100 text-yellow-700', icon: 'pause' }
|
||||
}
|
||||
|
||||
export default function UniCrawlerPage() {
|
||||
const [status, setStatus] = useState<OrchestratorStatus | null>(null)
|
||||
const [queue, setQueue] = useState<CrawlQueueItem[]>([])
|
||||
const [universities, setUniversities] = useState<University[]>([])
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [success, setSuccess] = useState<string | null>(null)
|
||||
|
||||
// Add to queue form
|
||||
const [selectedUniversity, setSelectedUniversity] = useState('')
|
||||
const [priority, setPriority] = useState(5)
|
||||
|
||||
// Fetch status
|
||||
const fetchStatus = useCallback(async () => {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=status`)
|
||||
if (res.ok) {
|
||||
const data = await res.json()
|
||||
setStatus(data)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch status:', err)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Fetch queue
|
||||
const fetchQueue = useCallback(async () => {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=queue`)
|
||||
if (res.ok) {
|
||||
const data = await res.json()
|
||||
setQueue(data.queue || [])
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch queue:', err)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Fetch universities (for dropdown)
|
||||
const fetchUniversities = useCallback(async () => {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=universities`)
|
||||
if (res.ok) {
|
||||
const data = await res.json()
|
||||
// Handle null/undefined universities array from API
|
||||
const unis = data.universities ?? data ?? []
|
||||
setUniversities(Array.isArray(unis) ? unis : [])
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch universities:', err)
|
||||
}
|
||||
}, [])
|
||||
|
||||
// Initial load and polling
|
||||
useEffect(() => {
|
||||
fetchStatus()
|
||||
fetchQueue()
|
||||
fetchUniversities()
|
||||
|
||||
// Poll every 5 seconds
|
||||
const interval = setInterval(() => {
|
||||
fetchStatus()
|
||||
fetchQueue()
|
||||
}, 5000)
|
||||
|
||||
return () => clearInterval(interval)
|
||||
}, [fetchStatus, fetchQueue, fetchUniversities])
|
||||
|
||||
// Start orchestrator
|
||||
const handleStart = async () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=start`, {
|
||||
method: 'POST'
|
||||
})
|
||||
if (res.ok) {
|
||||
setSuccess('Orchestrator gestartet')
|
||||
fetchStatus()
|
||||
} else {
|
||||
const data = await res.json()
|
||||
setError(data.error || 'Start fehlgeschlagen')
|
||||
}
|
||||
} catch (err) {
|
||||
setError('Verbindungsfehler')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop orchestrator
|
||||
const handleStop = async () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=stop`, {
|
||||
method: 'POST'
|
||||
})
|
||||
if (res.ok) {
|
||||
setSuccess('Orchestrator gestoppt')
|
||||
fetchStatus()
|
||||
} else {
|
||||
const data = await res.json()
|
||||
setError(data.error || 'Stop fehlgeschlagen')
|
||||
}
|
||||
} catch (err) {
|
||||
setError('Verbindungsfehler')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Add university to queue
|
||||
const handleAddToQueue = async (e: React.FormEvent) => {
|
||||
e.preventDefault()
|
||||
if (!selectedUniversity) return
|
||||
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=queue`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
university_id: selectedUniversity,
|
||||
priority: priority,
|
||||
initiated_by: 'admin-ui'
|
||||
})
|
||||
})
|
||||
if (res.ok) {
|
||||
setSuccess('Universitaet zur Queue hinzugefuegt')
|
||||
setSelectedUniversity('')
|
||||
setPriority(5)
|
||||
fetchQueue()
|
||||
} else {
|
||||
const data = await res.json()
|
||||
setError(data.error || 'Hinzufuegen fehlgeschlagen')
|
||||
}
|
||||
} catch (err) {
|
||||
setError('Verbindungsfehler')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from queue
|
||||
const handleRemove = async (universityId: string) => {
|
||||
if (!confirm('Wirklich aus der Queue entfernen?')) return
|
||||
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?university_id=${universityId}`, {
|
||||
method: 'DELETE'
|
||||
})
|
||||
if (res.ok) {
|
||||
fetchQueue()
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Remove failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
// Pause/Resume
|
||||
const handlePause = async (universityId: string) => {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=pause&university_id=${universityId}`, {
|
||||
method: 'POST'
|
||||
})
|
||||
if (res.ok) {
|
||||
fetchQueue()
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Pause failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
const handleResume = async (universityId: string) => {
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}?action=resume&university_id=${universityId}`, {
|
||||
method: 'POST'
|
||||
})
|
||||
if (res.ok) {
|
||||
fetchQueue()
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Resume failed:', err)
|
||||
}
|
||||
}
|
||||
|
||||
// Clear messages after 5 seconds
|
||||
useEffect(() => {
|
||||
if (success) {
|
||||
const timer = setTimeout(() => setSuccess(null), 5000)
|
||||
return () => clearTimeout(timer)
|
||||
}
|
||||
}, [success])
|
||||
|
||||
useEffect(() => {
|
||||
if (error) {
|
||||
const timer = setTimeout(() => setError(null), 5000)
|
||||
return () => clearTimeout(timer)
|
||||
}
|
||||
}, [error])
|
||||
|
||||
return (
|
||||
<AdminLayout
|
||||
title="University Crawler"
|
||||
description="Steuerung des mehrstufigen Crawling-Systems fuer Universitaeten"
|
||||
>
|
||||
{/* Messages */}
|
||||
{error && (
|
||||
<div className="mb-4 bg-red-50 border border-red-200 text-red-700 px-4 py-3 rounded-lg">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
{success && (
|
||||
<div className="mb-4 bg-green-50 border border-green-200 text-green-700 px-4 py-3 rounded-lg">
|
||||
{success}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="grid grid-cols-1 lg:grid-cols-3 gap-6">
|
||||
{/* Status Card */}
|
||||
<div className="lg:col-span-1">
|
||||
<div className="bg-white rounded-xl shadow-sm p-6 border border-gray-100">
|
||||
<h2 className="text-lg font-semibold text-gray-900 mb-4">Orchestrator Status</h2>
|
||||
|
||||
{/* Status Indicator */}
|
||||
<div className="flex items-center gap-3 mb-6">
|
||||
<div className={`w-4 h-4 rounded-full ${status?.is_running ? 'bg-green-500 animate-pulse' : 'bg-gray-400'}`} />
|
||||
<span className={`font-medium ${status?.is_running ? 'text-green-700' : 'text-gray-600'}`}>
|
||||
{status?.is_running ? 'Laeuft' : 'Gestoppt'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Control Buttons */}
|
||||
<div className="flex gap-3 mb-6">
|
||||
<button
|
||||
onClick={handleStart}
|
||||
disabled={loading || status?.is_running}
|
||||
className="flex-1 px-4 py-2 bg-green-600 text-white rounded-lg font-medium hover:bg-green-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
Start
|
||||
</button>
|
||||
<button
|
||||
onClick={handleStop}
|
||||
disabled={loading || !status?.is_running}
|
||||
className="flex-1 px-4 py-2 bg-red-600 text-white rounded-lg font-medium hover:bg-red-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
Stop
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Stats */}
|
||||
<div className="space-y-3">
|
||||
<div className="flex justify-between text-sm">
|
||||
<span className="text-gray-500">In Queue:</span>
|
||||
<span className="font-medium">{status?.queue_length || 0}</span>
|
||||
</div>
|
||||
<div className="flex justify-between text-sm">
|
||||
<span className="text-gray-500">Heute abgeschlossen:</span>
|
||||
<span className="font-medium text-green-600">{status?.completed_today || 0}</span>
|
||||
</div>
|
||||
<div className="flex justify-between text-sm">
|
||||
<span className="text-gray-500">Gesamt verarbeitet:</span>
|
||||
<span className="font-medium">{status?.total_processed || 0}</span>
|
||||
</div>
|
||||
{status?.last_activity && (
|
||||
<div className="flex justify-between text-sm">
|
||||
<span className="text-gray-500">Letzte Aktivitaet:</span>
|
||||
<span className="font-medium text-xs">
|
||||
{new Date(status.last_activity).toLocaleTimeString('de-DE')}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Current University */}
|
||||
{status?.current_university && (
|
||||
<div className="mt-6 pt-6 border-t border-gray-100">
|
||||
<h3 className="text-sm font-medium text-gray-500 mb-2">Aktuelle Verarbeitung</h3>
|
||||
<div className="bg-blue-50 rounded-lg p-3">
|
||||
<p className="font-medium text-blue-900">{status.current_university.university_name}</p>
|
||||
<div className="flex items-center gap-2 mt-2">
|
||||
<span className={`px-2 py-1 rounded text-xs font-medium ${phaseConfig[status.current_phase].color}`}>
|
||||
{phaseConfig[status.current_phase].label}
|
||||
</span>
|
||||
<span className="text-xs text-blue-600">
|
||||
{status.current_university.progress_percent}%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Add to Queue Form */}
|
||||
<div className="bg-white rounded-xl shadow-sm p-6 border border-gray-100 mt-6">
|
||||
<h2 className="text-lg font-semibold text-gray-900 mb-4">Zur Queue hinzufuegen</h2>
|
||||
<form onSubmit={handleAddToQueue} className="space-y-4">
|
||||
<div>
|
||||
<label className="block text-sm font-medium text-gray-700 mb-1">
|
||||
Universitaet
|
||||
</label>
|
||||
<select
|
||||
value={selectedUniversity}
|
||||
onChange={(e) => setSelectedUniversity(e.target.value)}
|
||||
className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
<option value="">Waehlen...</option>
|
||||
{universities.map((uni) => (
|
||||
<option key={uni.id} value={uni.id}>
|
||||
{uni.name} {uni.short_name && `(${uni.short_name})`}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<div>
|
||||
<label className="block text-sm font-medium text-gray-700 mb-1">
|
||||
Prioritaet (1-10)
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={10}
|
||||
value={priority}
|
||||
onChange={(e) => setPriority(Number(e.target.value))}
|
||||
className="w-full px-3 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
/>
|
||||
<p className="text-xs text-gray-500 mt-1">Hoeher = dringender</p>
|
||||
</div>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={loading || !selectedUniversity}
|
||||
className="w-full px-4 py-2 bg-blue-600 text-white rounded-lg font-medium hover:bg-blue-700 disabled:opacity-50 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
Hinzufuegen
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Queue List */}
|
||||
<div className="lg:col-span-2">
|
||||
<div className="bg-white rounded-xl shadow-sm border border-gray-100">
|
||||
<div className="px-6 py-4 border-b border-gray-100">
|
||||
<h2 className="text-lg font-semibold text-gray-900">Crawl Queue ({queue.length})</h2>
|
||||
</div>
|
||||
|
||||
{queue.length === 0 ? (
|
||||
<div className="px-6 py-12 text-center text-gray-500">
|
||||
Queue ist leer. Fuege Universitaeten hinzu, um das Crawling zu starten.
|
||||
</div>
|
||||
) : (
|
||||
<div className="divide-y divide-gray-100">
|
||||
{queue.map((item) => (
|
||||
<div key={item.id} className="px-6 py-4 hover:bg-gray-50 transition-colors">
|
||||
<div className="flex items-start justify-between gap-4">
|
||||
<div className="flex-1">
|
||||
{/* Header */}
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-sm font-medium text-gray-400">#{item.queue_position || '-'}</span>
|
||||
<h3 className="font-medium text-gray-900">{item.university_name}</h3>
|
||||
<span className="text-sm text-gray-500">({item.university_short})</span>
|
||||
</div>
|
||||
|
||||
{/* Phase Badge */}
|
||||
<div className="flex items-center gap-2 mt-2">
|
||||
<span className={`px-2 py-1 rounded text-xs font-medium ${phaseConfig[item.current_phase].color}`}>
|
||||
{phaseConfig[item.current_phase].label}
|
||||
</span>
|
||||
<span className="text-xs text-gray-500">
|
||||
Prioritaet: {item.priority}
|
||||
</span>
|
||||
{item.retry_count > 0 && (
|
||||
<span className="text-xs text-orange-600">
|
||||
Versuch {item.retry_count}/{item.max_retries}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Progress Bar */}
|
||||
<div className="mt-3">
|
||||
<div className="h-2 bg-gray-100 rounded-full overflow-hidden">
|
||||
<div
|
||||
className="h-full bg-blue-500 transition-all duration-500"
|
||||
style={{ width: `${item.progress_percent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex justify-between text-xs text-gray-500 mt-1">
|
||||
<span>{item.progress_percent}%</span>
|
||||
<span>
|
||||
{item.discovery_count} Disc / {item.professors_count} Prof / {item.staff_count} Staff / {item.publications_count} Pub
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Phase Checkmarks */}
|
||||
<div className="flex gap-4 mt-3 text-xs">
|
||||
<span className={item.discovery_completed ? 'text-green-600' : 'text-gray-400'}>
|
||||
{item.discovery_completed ? 'Discovery' : 'Discovery'}
|
||||
</span>
|
||||
<span className={item.professors_completed ? 'text-green-600' : 'text-gray-400'}>
|
||||
{item.professors_completed ? 'Professoren' : 'Professoren'}
|
||||
</span>
|
||||
<span className={item.all_staff_completed ? 'text-green-600' : 'text-gray-400'}>
|
||||
{item.all_staff_completed ? 'Mitarbeiter' : 'Mitarbeiter'}
|
||||
</span>
|
||||
<span className={item.publications_completed ? 'text-green-600' : 'text-gray-400'}>
|
||||
{item.publications_completed ? 'Publikationen' : 'Publikationen'}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Error Message */}
|
||||
{item.last_error && (
|
||||
<p className="mt-2 text-xs text-red-600 bg-red-50 px-2 py-1 rounded">
|
||||
{item.last_error}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
<div className="flex flex-col gap-2">
|
||||
{item.current_phase === 'paused' ? (
|
||||
<button
|
||||
onClick={() => handleResume(item.university_id)}
|
||||
className="px-3 py-1 text-xs bg-green-100 text-green-700 rounded hover:bg-green-200 transition-colors"
|
||||
>
|
||||
Fortsetzen
|
||||
</button>
|
||||
) : item.current_phase !== 'completed' && item.current_phase !== 'failed' ? (
|
||||
<button
|
||||
onClick={() => handlePause(item.university_id)}
|
||||
className="px-3 py-1 text-xs bg-yellow-100 text-yellow-700 rounded hover:bg-yellow-200 transition-colors"
|
||||
>
|
||||
Pausieren
|
||||
</button>
|
||||
) : null}
|
||||
<button
|
||||
onClick={() => handleRemove(item.university_id)}
|
||||
className="px-3 py-1 text-xs bg-red-100 text-red-700 rounded hover:bg-red-200 transition-colors"
|
||||
>
|
||||
Entfernen
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</AdminLayout>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user