Files
breakpilot-lehrer/admin-lehrer/components/ocr-pipeline/StepGroundTruth.tsx
Benjamin Admin 1cc69d6b5e
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 29s
CI / test-go-edu-search (push) Successful in 27s
CI / test-python-klausur (push) Failing after 2m4s
CI / test-python-agent-core (push) Successful in 19s
CI / test-nodejs-website (push) Successful in 19s
feat: OCR pipeline step 8 — validation view with image detection & generation
Replaces the stub StepGroundTruth with a full side-by-side Original vs
Reconstruction view. Adds VLM-based image region detection (qwen2.5vl),
mflux image generation proxy, sync scroll/zoom, manual region drawing,
and score/notes persistence.

New backend endpoints: detect-images, generate-image, validate, get validation.
New standalone mflux-service (scripts/mflux-service.py) for Metal GPU generation.
Dockerfile.base: adds fonts-liberation (Apache-2.0).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 10:40:37 +01:00

584 lines
22 KiB
TypeScript

'use client'
import { useCallback, useEffect, useRef, useState } from 'react'
import type {
GridCell, ColumnMeta, ImageRegion, ImageStyle,
} from '@/app/(admin)/ai/ocr-pipeline/types'
import { IMAGE_STYLES as STYLES } from '@/app/(admin)/ai/ocr-pipeline/types'
const KLAUSUR_API = '/klausur-api'
const COL_TYPE_COLORS: Record<string, string> = {
column_en: '#3b82f6',
column_de: '#22c55e',
column_example: '#f97316',
column_text: '#a855f7',
page_ref: '#06b6d4',
column_marker: '#6b7280',
}
interface StepGroundTruthProps {
sessionId: string | null
onNext: () => void
}
interface SessionData {
cells: GridCell[]
columnsUsed: ColumnMeta[]
imageWidth: number
imageHeight: number
originalImageUrl: string
}
export function StepGroundTruth({ sessionId, onNext }: StepGroundTruthProps) {
const [status, setStatus] = useState<'loading' | 'ready' | 'saving' | 'saved' | 'error'>('loading')
const [error, setError] = useState('')
const [session, setSession] = useState<SessionData | null>(null)
const [imageRegions, setImageRegions] = useState<(ImageRegion & { generating?: boolean })[]>([])
const [detecting, setDetecting] = useState(false)
const [zoom, setZoom] = useState(100)
const [syncScroll, setSyncScroll] = useState(true)
const [notes, setNotes] = useState('')
const [score, setScore] = useState<number | null>(null)
const [drawingRegion, setDrawingRegion] = useState(false)
const [dragStart, setDragStart] = useState<{ x: number; y: number } | null>(null)
const [dragEnd, setDragEnd] = useState<{ x: number; y: number } | null>(null)
const leftPanelRef = useRef<HTMLDivElement>(null)
const rightPanelRef = useRef<HTMLDivElement>(null)
// Load session data
useEffect(() => {
if (!sessionId) return
loadSessionData()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [sessionId])
const loadSessionData = async () => {
if (!sessionId) return
setStatus('loading')
try {
const resp = await fetch(`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}`)
if (!resp.ok) throw new Error(`Failed to load session: ${resp.status}`)
const data = await resp.json()
const wordResult = data.word_result || {}
setSession({
cells: wordResult.cells || [],
columnsUsed: wordResult.columns_used || [],
imageWidth: wordResult.image_width || data.image_width || 800,
imageHeight: wordResult.image_height || data.image_height || 600,
originalImageUrl: data.original_image_url || `${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/image/original`,
})
// Load existing validation data
const valResp = await fetch(`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/validation`)
if (valResp.ok) {
const valData = await valResp.json()
const validation = valData.validation
if (validation) {
setImageRegions(validation.image_regions || [])
setNotes(validation.notes || '')
setScore(validation.score ?? null)
}
}
setStatus('ready')
} catch (e) {
setError(e instanceof Error ? e.message : String(e))
setStatus('error')
}
}
// Sync scroll between panels
const handleScroll = useCallback((source: 'left' | 'right') => {
if (!syncScroll) return
const from = source === 'left' ? leftPanelRef.current : rightPanelRef.current
const to = source === 'left' ? rightPanelRef.current : leftPanelRef.current
if (from && to) {
to.scrollTop = from.scrollTop
to.scrollLeft = from.scrollLeft
}
}, [syncScroll])
// Detect images via VLM
const handleDetectImages = async () => {
if (!sessionId) return
setDetecting(true)
try {
const resp = await fetch(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/detect-images`,
{ method: 'POST' }
)
if (!resp.ok) throw new Error(`Detection failed: ${resp.status}`)
const data = await resp.json()
setImageRegions(data.regions || [])
} catch (e) {
setError(e instanceof Error ? e.message : String(e))
} finally {
setDetecting(false)
}
}
// Generate image for a region
const handleGenerateImage = async (index: number) => {
if (!sessionId) return
const region = imageRegions[index]
if (!region) return
setImageRegions(prev => prev.map((r, i) => i === index ? { ...r, generating: true } : r))
try {
const resp = await fetch(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/generate-image`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
region_index: index,
prompt: region.prompt,
style: region.style,
}),
}
)
if (!resp.ok) throw new Error(`Generation failed: ${resp.status}`)
const data = await resp.json()
setImageRegions(prev => prev.map((r, i) =>
i === index ? { ...r, image_b64: data.image_b64, generating: false } : r
))
} catch (e) {
setImageRegions(prev => prev.map((r, i) => i === index ? { ...r, generating: false } : r))
setError(e instanceof Error ? e.message : String(e))
}
}
// Save validation
const handleSave = async () => {
if (!sessionId) return
setStatus('saving')
try {
const resp = await fetch(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/validate`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ notes, score }),
}
)
if (!resp.ok) throw new Error(`Save failed: ${resp.status}`)
setStatus('saved')
} catch (e) {
setError(e instanceof Error ? e.message : String(e))
setStatus('error')
}
}
// Handle manual region drawing on reconstruction
const handleReconMouseDown = (e: React.MouseEvent<HTMLDivElement>) => {
if (!drawingRegion) return
const rect = e.currentTarget.getBoundingClientRect()
const x = ((e.clientX - rect.left) / rect.width) * 100
const y = ((e.clientY - rect.top) / rect.height) * 100
setDragStart({ x, y })
setDragEnd({ x, y })
}
const handleReconMouseMove = (e: React.MouseEvent<HTMLDivElement>) => {
if (!dragStart) return
const rect = e.currentTarget.getBoundingClientRect()
const x = ((e.clientX - rect.left) / rect.width) * 100
const y = ((e.clientY - rect.top) / rect.height) * 100
setDragEnd({ x, y })
}
const handleReconMouseUp = () => {
if (!dragStart || !dragEnd) return
const x = Math.min(dragStart.x, dragEnd.x)
const y = Math.min(dragStart.y, dragEnd.y)
const w = Math.abs(dragEnd.x - dragStart.x)
const h = Math.abs(dragEnd.y - dragStart.y)
if (w > 2 && h > 2) {
setImageRegions(prev => [...prev, {
bbox_pct: { x, y, w, h },
prompt: '',
description: 'Manually selected region',
image_b64: null,
style: 'educational' as ImageStyle,
}])
}
setDragStart(null)
setDragEnd(null)
setDrawingRegion(false)
}
const handleRemoveRegion = (index: number) => {
setImageRegions(prev => prev.filter((_, i) => i !== index))
}
if (status === 'loading') {
return (
<div className="flex items-center justify-center py-16">
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-teal-500 mr-3" />
<span className="text-gray-500 dark:text-gray-400">Session wird geladen...</span>
</div>
)
}
if (status === 'error' && !session) {
return (
<div className="text-center py-16">
<p className="text-red-500">{error}</p>
<button onClick={loadSessionData} className="mt-4 px-4 py-2 bg-teal-600 text-white rounded hover:bg-teal-700">
Erneut laden
</button>
</div>
)
}
if (!session) return null
const aspect = session.imageHeight / session.imageWidth
return (
<div className="space-y-4">
{/* Header / Controls */}
<div className="flex items-center justify-between flex-wrap gap-2">
<h3 className="text-lg font-medium text-gray-800 dark:text-gray-200">
Validierung Original vs. Rekonstruktion
</h3>
<div className="flex items-center gap-3">
<button
onClick={handleDetectImages}
disabled={detecting}
className="px-3 py-1.5 text-sm bg-indigo-600 text-white rounded hover:bg-indigo-700 disabled:opacity-50"
>
{detecting ? 'Erkennung laeuft...' : 'Bilder erkennen'}
</button>
<label className="flex items-center gap-1.5 text-sm text-gray-600 dark:text-gray-400">
<input
type="checkbox"
checked={syncScroll}
onChange={e => setSyncScroll(e.target.checked)}
className="rounded"
/>
Sync Scroll
</label>
<div className="flex items-center gap-1.5">
<button onClick={() => setZoom(z => Math.max(50, z - 25))} className="px-2 py-1 text-sm border rounded dark:border-gray-600 hover:bg-gray-100 dark:hover:bg-gray-700">-</button>
<span className="text-sm text-gray-600 dark:text-gray-400 w-12 text-center">{zoom}%</span>
<button onClick={() => setZoom(z => Math.min(200, z + 25))} className="px-2 py-1 text-sm border rounded dark:border-gray-600 hover:bg-gray-100 dark:hover:bg-gray-700">+</button>
</div>
</div>
</div>
{error && (
<div className="p-2 bg-red-50 dark:bg-red-900/20 text-red-600 dark:text-red-400 text-sm rounded">
{error}
<button onClick={() => setError('')} className="ml-2 underline">Schliessen</button>
</div>
)}
{/* Side-by-side panels */}
<div className="grid grid-cols-2 gap-4" style={{ height: 'calc(100vh - 380px)', minHeight: 400 }}>
{/* Left: Original */}
<div className="border rounded-lg dark:border-gray-700 overflow-hidden flex flex-col">
<div className="px-3 py-1.5 bg-gray-50 dark:bg-gray-800 text-sm font-medium text-gray-600 dark:text-gray-400 border-b dark:border-gray-700">
Original
</div>
<div
ref={leftPanelRef}
className="flex-1 overflow-auto"
onScroll={() => handleScroll('left')}
>
<div style={{ width: `${zoom}%`, minWidth: '100%' }}>
<img
src={session.originalImageUrl}
alt="Original"
className="w-full h-auto"
draggable={false}
/>
</div>
</div>
</div>
{/* Right: Reconstruction */}
<div className="border rounded-lg dark:border-gray-700 overflow-hidden flex flex-col">
<div className="px-3 py-1.5 bg-gray-50 dark:bg-gray-800 text-sm font-medium text-gray-600 dark:text-gray-400 border-b dark:border-gray-700 flex items-center justify-between">
<span>Rekonstruktion</span>
<button
onClick={() => setDrawingRegion(!drawingRegion)}
className={`text-xs px-2 py-0.5 rounded ${drawingRegion ? 'bg-indigo-600 text-white' : 'bg-gray-200 dark:bg-gray-700 text-gray-600 dark:text-gray-400'}`}
>
{drawingRegion ? 'Region zeichnen...' : '+ Region'}
</button>
</div>
<div
ref={rightPanelRef}
className="flex-1 overflow-auto"
onScroll={() => handleScroll('right')}
>
<div style={{ width: `${zoom}%`, minWidth: '100%' }}>
{/* Reconstruction container */}
<div
className="relative bg-white"
style={{
paddingBottom: `${aspect * 100}%`,
cursor: drawingRegion ? 'crosshair' : 'default',
}}
onMouseDown={handleReconMouseDown}
onMouseMove={handleReconMouseMove}
onMouseUp={handleReconMouseUp}
>
{/* Column background stripes */}
{session.columnsUsed.map((col, i) => {
const color = COL_TYPE_COLORS[col.type] || '#9ca3af'
return (
<div
key={`col-${i}`}
className="absolute top-0 bottom-0"
style={{
left: `${(col.x / session.imageWidth) * 100}%`,
width: `${(col.width / session.imageWidth) * 100}%`,
backgroundColor: color,
opacity: 0.06,
}}
/>
)
})}
{/* Row separator lines — derive from cells */}
{(() => {
const rowYs = new Set<number>()
for (const cell of session.cells) {
if (cell.col_index === 0 && cell.bbox_pct) {
rowYs.add(cell.bbox_pct.y)
}
}
return Array.from(rowYs).map((y, i) => (
<div
key={`row-${i}`}
className="absolute left-0 right-0"
style={{
top: `${y}%`,
height: '1px',
backgroundColor: 'rgba(0,0,0,0.08)',
}}
/>
))
})()}
{/* Cell texts */}
{session.cells.map(cell => {
if (!cell.bbox_pct || !cell.text) return null
const color = COL_TYPE_COLORS[cell.col_type] || '#374151'
return (
<span
key={cell.cell_id}
className="absolute text-[0.6em] leading-tight overflow-hidden"
style={{
left: `${cell.bbox_pct.x}%`,
top: `${cell.bbox_pct.y}%`,
width: `${cell.bbox_pct.w}%`,
height: `${cell.bbox_pct.h}%`,
color,
fontFamily: "'Liberation Sans', 'DejaVu Sans', sans-serif",
display: 'flex',
alignItems: 'center',
padding: '0 1px',
}}
title={`${cell.cell_id}: ${cell.text}`}
>
{cell.text}
</span>
)
})}
{/* Generated images at region positions */}
{imageRegions.map((region, i) => (
<div
key={`region-${i}`}
className="absolute border-2 border-dashed border-indigo-400"
style={{
left: `${region.bbox_pct.x}%`,
top: `${region.bbox_pct.y}%`,
width: `${region.bbox_pct.w}%`,
height: `${region.bbox_pct.h}%`,
}}
>
{region.image_b64 ? (
<img src={region.image_b64} alt={region.description} className="w-full h-full object-cover" />
) : (
<div className="w-full h-full flex items-center justify-center bg-indigo-50/50 text-indigo-400 text-[0.5em]">
{region.generating ? '...' : `Bild ${i + 1}`}
</div>
)}
</div>
))}
{/* Drawing rectangle */}
{dragStart && dragEnd && (
<div
className="absolute border-2 border-dashed border-red-500 bg-red-100/20 pointer-events-none"
style={{
left: `${Math.min(dragStart.x, dragEnd.x)}%`,
top: `${Math.min(dragStart.y, dragEnd.y)}%`,
width: `${Math.abs(dragEnd.x - dragStart.x)}%`,
height: `${Math.abs(dragEnd.y - dragStart.y)}%`,
}}
/>
)}
</div>
</div>
</div>
</div>
</div>
{/* Image regions panel */}
{imageRegions.length > 0 && (
<div className="border rounded-lg dark:border-gray-700 p-4">
<h4 className="text-sm font-medium text-gray-700 dark:text-gray-300 mb-3">
Bildbereiche ({imageRegions.length} gefunden)
</h4>
<div className="space-y-3">
{imageRegions.map((region, i) => (
<div key={i} className="flex items-start gap-3 p-3 bg-gray-50 dark:bg-gray-800 rounded-lg">
{/* Preview thumbnail */}
<div className="w-16 h-16 flex-shrink-0 border rounded dark:border-gray-600 overflow-hidden bg-white">
{region.image_b64 ? (
<img src={region.image_b64} alt="" className="w-full h-full object-cover" />
) : (
<div className="w-full h-full flex items-center justify-center text-gray-400 text-xs">
{Math.round(region.bbox_pct.w)}x{Math.round(region.bbox_pct.h)}%
</div>
)}
</div>
{/* Prompt + controls */}
<div className="flex-1 min-w-0 space-y-2">
<div className="flex items-center gap-2">
<span className="text-xs text-gray-500 dark:text-gray-400 flex-shrink-0">
Bereich {i + 1}:
</span>
<input
type="text"
value={region.prompt}
onChange={e => {
setImageRegions(prev => prev.map((r, j) =>
j === i ? { ...r, prompt: e.target.value } : r
))
}}
placeholder="Beschreibung / Prompt..."
className="flex-1 text-sm px-2 py-1 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
</div>
<div className="flex items-center gap-2">
<select
value={region.style}
onChange={e => {
setImageRegions(prev => prev.map((r, j) =>
j === i ? { ...r, style: e.target.value as ImageStyle } : r
))
}}
className="text-sm px-2 py-1 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
>
{STYLES.map(s => (
<option key={s.value} value={s.value}>{s.label}</option>
))}
</select>
<button
onClick={() => handleGenerateImage(i)}
disabled={!!region.generating || !region.prompt}
className="px-3 py-1 text-sm bg-teal-600 text-white rounded hover:bg-teal-700 disabled:opacity-50"
>
{region.generating ? 'Generiere...' : 'Generieren'}
</button>
<button
onClick={() => handleRemoveRegion(i)}
className="px-2 py-1 text-sm text-red-600 hover:bg-red-50 dark:hover:bg-red-900/20 rounded"
>
Entfernen
</button>
</div>
{region.description && region.description !== region.prompt && (
<p className="text-xs text-gray-400">{region.description}</p>
)}
</div>
</div>
))}
</div>
</div>
)}
{/* Notes and score */}
<div className="border rounded-lg dark:border-gray-700 p-4 space-y-3">
<div className="flex items-center gap-4">
<label className="text-sm font-medium text-gray-700 dark:text-gray-300">
Bewertung (1-10):
</label>
<input
type="number"
min={1}
max={10}
value={score ?? ''}
onChange={e => setScore(e.target.value ? parseInt(e.target.value) : null)}
className="w-20 text-sm px-2 py-1 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
<div className="flex gap-1">
{[1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map(v => (
<button
key={v}
onClick={() => setScore(v)}
className={`w-7 h-7 text-xs rounded ${score === v ? 'bg-teal-600 text-white' : 'bg-gray-100 dark:bg-gray-700 text-gray-600 dark:text-gray-400 hover:bg-gray-200 dark:hover:bg-gray-600'}`}
>
{v}
</button>
))}
</div>
</div>
<div>
<label className="text-sm font-medium text-gray-700 dark:text-gray-300 block mb-1">
Notizen:
</label>
<textarea
value={notes}
onChange={e => setNotes(e.target.value)}
rows={3}
placeholder="Anmerkungen zur Qualitaet der Rekonstruktion..."
className="w-full text-sm px-3 py-2 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
</div>
</div>
{/* Actions */}
<div className="flex items-center justify-between">
<div className="text-sm text-gray-500 dark:text-gray-400">
{status === 'saved' && <span className="text-green-600 dark:text-green-400">Validierung gespeichert</span>}
{status === 'saving' && <span>Speichere...</span>}
</div>
<div className="flex items-center gap-3">
<button
onClick={handleSave}
disabled={status === 'saving'}
className="px-4 py-2 text-sm bg-gray-600 text-white rounded hover:bg-gray-700 disabled:opacity-50"
>
Speichern
</button>
<button
onClick={async () => {
await handleSave()
onNext()
}}
disabled={status === 'saving'}
className="px-4 py-2 text-sm bg-teal-600 text-white rounded hover:bg-teal-700 disabled:opacity-50"
>
Abschliessen
</button>
</div>
</div>
</div>
)
}