feat: OCR pipeline step 8 — validation view with image detection & generation
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 29s
CI / test-go-edu-search (push) Successful in 27s
CI / test-python-klausur (push) Failing after 2m4s
CI / test-python-agent-core (push) Successful in 19s
CI / test-nodejs-website (push) Successful in 19s

Replaces the stub StepGroundTruth with a full side-by-side Original vs
Reconstruction view. Adds VLM-based image region detection (qwen2.5vl),
mflux image generation proxy, sync scroll/zoom, manual region drawing,
and score/notes persistence.

New backend endpoints: detect-images, generate-image, validate, get validation.
New standalone mflux-service (scripts/mflux-service.py) for Metal GPU generation.
Dockerfile.base: adds fonts-liberation (Apache-2.0).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-05 10:40:37 +01:00
parent 293e7914d8
commit 1cc69d6b5e
7 changed files with 1284 additions and 69 deletions

View File

@@ -313,7 +313,7 @@ export default function OcrPipelinePage() {
case 6:
return <StepReconstruction sessionId={sessionId} onNext={handleNext} />
case 7:
return <StepGroundTruth />
return <StepGroundTruth sessionId={sessionId} onNext={handleNext} />
default:
return null
}

View File

@@ -264,6 +264,24 @@ export interface WordGroundTruth {
notes?: string
}
export interface ImageRegion {
bbox_pct: { x: number; y: number; w: number; h: number }
prompt: string
description: string
image_b64: string | null
style: 'educational' | 'cartoon' | 'sketch' | 'clipart' | 'realistic'
}
export type ImageStyle = ImageRegion['style']
export const IMAGE_STYLES: { value: ImageStyle; label: string }[] = [
{ value: 'educational', label: 'Lehrbuch' },
{ value: 'cartoon', label: 'Cartoon' },
{ value: 'sketch', label: 'Skizze' },
{ value: 'clipart', label: 'Clipart' },
{ value: 'realistic', label: 'Realistisch' },
]
export const PIPELINE_STEPS: PipelineStep[] = [
{ id: 'deskew', name: 'Begradigung', icon: '📐', status: 'pending' },
{ id: 'dewarp', name: 'Entzerrung', icon: '🔧', status: 'pending' },

View File

@@ -1,18 +1,582 @@
'use client'
export function StepGroundTruth() {
import { useCallback, useEffect, useRef, useState } from 'react'
import type {
GridCell, ColumnMeta, ImageRegion, ImageStyle,
} from '@/app/(admin)/ai/ocr-pipeline/types'
import { IMAGE_STYLES as STYLES } from '@/app/(admin)/ai/ocr-pipeline/types'
const KLAUSUR_API = '/klausur-api'
const COL_TYPE_COLORS: Record<string, string> = {
column_en: '#3b82f6',
column_de: '#22c55e',
column_example: '#f97316',
column_text: '#a855f7',
page_ref: '#06b6d4',
column_marker: '#6b7280',
}
interface StepGroundTruthProps {
sessionId: string | null
onNext: () => void
}
interface SessionData {
cells: GridCell[]
columnsUsed: ColumnMeta[]
imageWidth: number
imageHeight: number
originalImageUrl: string
}
export function StepGroundTruth({ sessionId, onNext }: StepGroundTruthProps) {
const [status, setStatus] = useState<'loading' | 'ready' | 'saving' | 'saved' | 'error'>('loading')
const [error, setError] = useState('')
const [session, setSession] = useState<SessionData | null>(null)
const [imageRegions, setImageRegions] = useState<(ImageRegion & { generating?: boolean })[]>([])
const [detecting, setDetecting] = useState(false)
const [zoom, setZoom] = useState(100)
const [syncScroll, setSyncScroll] = useState(true)
const [notes, setNotes] = useState('')
const [score, setScore] = useState<number | null>(null)
const [drawingRegion, setDrawingRegion] = useState(false)
const [dragStart, setDragStart] = useState<{ x: number; y: number } | null>(null)
const [dragEnd, setDragEnd] = useState<{ x: number; y: number } | null>(null)
const leftPanelRef = useRef<HTMLDivElement>(null)
const rightPanelRef = useRef<HTMLDivElement>(null)
// Load session data
useEffect(() => {
if (!sessionId) return
loadSessionData()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [sessionId])
const loadSessionData = async () => {
if (!sessionId) return
setStatus('loading')
try {
const resp = await fetch(`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}`)
if (!resp.ok) throw new Error(`Failed to load session: ${resp.status}`)
const data = await resp.json()
const wordResult = data.word_result || {}
setSession({
cells: wordResult.cells || [],
columnsUsed: wordResult.columns_used || [],
imageWidth: wordResult.image_width || data.image_width || 800,
imageHeight: wordResult.image_height || data.image_height || 600,
originalImageUrl: data.original_image_url || `${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/image/original`,
})
// Load existing validation data
const valResp = await fetch(`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/validation`)
if (valResp.ok) {
const valData = await valResp.json()
const validation = valData.validation
if (validation) {
setImageRegions(validation.image_regions || [])
setNotes(validation.notes || '')
setScore(validation.score ?? null)
}
}
setStatus('ready')
} catch (e) {
setError(e instanceof Error ? e.message : String(e))
setStatus('error')
}
}
// Sync scroll between panels
const handleScroll = useCallback((source: 'left' | 'right') => {
if (!syncScroll) return
const from = source === 'left' ? leftPanelRef.current : rightPanelRef.current
const to = source === 'left' ? rightPanelRef.current : leftPanelRef.current
if (from && to) {
to.scrollTop = from.scrollTop
to.scrollLeft = from.scrollLeft
}
}, [syncScroll])
// Detect images via VLM
const handleDetectImages = async () => {
if (!sessionId) return
setDetecting(true)
try {
const resp = await fetch(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/detect-images`,
{ method: 'POST' }
)
if (!resp.ok) throw new Error(`Detection failed: ${resp.status}`)
const data = await resp.json()
setImageRegions(data.regions || [])
} catch (e) {
setError(e instanceof Error ? e.message : String(e))
} finally {
setDetecting(false)
}
}
// Generate image for a region
const handleGenerateImage = async (index: number) => {
if (!sessionId) return
const region = imageRegions[index]
if (!region) return
setImageRegions(prev => prev.map((r, i) => i === index ? { ...r, generating: true } : r))
try {
const resp = await fetch(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/generate-image`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
region_index: index,
prompt: region.prompt,
style: region.style,
}),
}
)
if (!resp.ok) throw new Error(`Generation failed: ${resp.status}`)
const data = await resp.json()
setImageRegions(prev => prev.map((r, i) =>
i === index ? { ...r, image_b64: data.image_b64, generating: false } : r
))
} catch (e) {
setImageRegions(prev => prev.map((r, i) => i === index ? { ...r, generating: false } : r))
setError(e instanceof Error ? e.message : String(e))
}
}
// Save validation
const handleSave = async () => {
if (!sessionId) return
setStatus('saving')
try {
const resp = await fetch(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/validate`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ notes, score }),
}
)
if (!resp.ok) throw new Error(`Save failed: ${resp.status}`)
setStatus('saved')
} catch (e) {
setError(e instanceof Error ? e.message : String(e))
setStatus('error')
}
}
// Handle manual region drawing on reconstruction
const handleReconMouseDown = (e: React.MouseEvent<HTMLDivElement>) => {
if (!drawingRegion) return
const rect = e.currentTarget.getBoundingClientRect()
const x = ((e.clientX - rect.left) / rect.width) * 100
const y = ((e.clientY - rect.top) / rect.height) * 100
setDragStart({ x, y })
setDragEnd({ x, y })
}
const handleReconMouseMove = (e: React.MouseEvent<HTMLDivElement>) => {
if (!dragStart) return
const rect = e.currentTarget.getBoundingClientRect()
const x = ((e.clientX - rect.left) / rect.width) * 100
const y = ((e.clientY - rect.top) / rect.height) * 100
setDragEnd({ x, y })
}
const handleReconMouseUp = () => {
if (!dragStart || !dragEnd) return
const x = Math.min(dragStart.x, dragEnd.x)
const y = Math.min(dragStart.y, dragEnd.y)
const w = Math.abs(dragEnd.x - dragStart.x)
const h = Math.abs(dragEnd.y - dragStart.y)
if (w > 2 && h > 2) {
setImageRegions(prev => [...prev, {
bbox_pct: { x, y, w, h },
prompt: '',
description: 'Manually selected region',
image_b64: null,
style: 'educational' as ImageStyle,
}])
}
setDragStart(null)
setDragEnd(null)
setDrawingRegion(false)
}
const handleRemoveRegion = (index: number) => {
setImageRegions(prev => prev.filter((_, i) => i !== index))
}
if (status === 'loading') {
return (
<div className="flex items-center justify-center py-16">
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-teal-500 mr-3" />
<span className="text-gray-500 dark:text-gray-400">Session wird geladen...</span>
</div>
)
}
if (status === 'error' && !session) {
return (
<div className="text-center py-16">
<p className="text-red-500">{error}</p>
<button onClick={loadSessionData} className="mt-4 px-4 py-2 bg-teal-600 text-white rounded hover:bg-teal-700">
Erneut laden
</button>
</div>
)
}
if (!session) return null
const aspect = session.imageHeight / session.imageWidth
return (
<div className="flex flex-col items-center justify-center py-16 text-center">
<div className="text-5xl mb-4"></div>
<h3 className="text-lg font-medium text-gray-700 dark:text-gray-300 mb-2">
Schritt 7: Ground Truth Validierung
</h3>
<p className="text-gray-500 dark:text-gray-400 max-w-md">
Gesamtpruefung der rekonstruierten Seite gegen das Original.
Dieser Schritt wird in einer zukuenftigen Version implementiert.
</p>
<div className="mt-6 px-4 py-2 bg-amber-100 dark:bg-amber-900/30 text-amber-700 dark:text-amber-400 rounded-full text-sm font-medium">
Kommt bald
<div className="space-y-4">
{/* Header / Controls */}
<div className="flex items-center justify-between flex-wrap gap-2">
<h3 className="text-lg font-medium text-gray-800 dark:text-gray-200">
Validierung Original vs. Rekonstruktion
</h3>
<div className="flex items-center gap-3">
<button
onClick={handleDetectImages}
disabled={detecting}
className="px-3 py-1.5 text-sm bg-indigo-600 text-white rounded hover:bg-indigo-700 disabled:opacity-50"
>
{detecting ? 'Erkennung laeuft...' : 'Bilder erkennen'}
</button>
<label className="flex items-center gap-1.5 text-sm text-gray-600 dark:text-gray-400">
<input
type="checkbox"
checked={syncScroll}
onChange={e => setSyncScroll(e.target.checked)}
className="rounded"
/>
Sync Scroll
</label>
<div className="flex items-center gap-1.5">
<button onClick={() => setZoom(z => Math.max(50, z - 25))} className="px-2 py-1 text-sm border rounded dark:border-gray-600 hover:bg-gray-100 dark:hover:bg-gray-700">-</button>
<span className="text-sm text-gray-600 dark:text-gray-400 w-12 text-center">{zoom}%</span>
<button onClick={() => setZoom(z => Math.min(200, z + 25))} className="px-2 py-1 text-sm border rounded dark:border-gray-600 hover:bg-gray-100 dark:hover:bg-gray-700">+</button>
</div>
</div>
</div>
{error && (
<div className="p-2 bg-red-50 dark:bg-red-900/20 text-red-600 dark:text-red-400 text-sm rounded">
{error}
<button onClick={() => setError('')} className="ml-2 underline">Schliessen</button>
</div>
)}
{/* Side-by-side panels */}
<div className="grid grid-cols-2 gap-4" style={{ height: 'calc(100vh - 380px)', minHeight: 400 }}>
{/* Left: Original */}
<div className="border rounded-lg dark:border-gray-700 overflow-hidden flex flex-col">
<div className="px-3 py-1.5 bg-gray-50 dark:bg-gray-800 text-sm font-medium text-gray-600 dark:text-gray-400 border-b dark:border-gray-700">
Original
</div>
<div
ref={leftPanelRef}
className="flex-1 overflow-auto"
onScroll={() => handleScroll('left')}
>
<div style={{ width: `${zoom}%`, minWidth: '100%' }}>
<img
src={session.originalImageUrl}
alt="Original"
className="w-full h-auto"
draggable={false}
/>
</div>
</div>
</div>
{/* Right: Reconstruction */}
<div className="border rounded-lg dark:border-gray-700 overflow-hidden flex flex-col">
<div className="px-3 py-1.5 bg-gray-50 dark:bg-gray-800 text-sm font-medium text-gray-600 dark:text-gray-400 border-b dark:border-gray-700 flex items-center justify-between">
<span>Rekonstruktion</span>
<button
onClick={() => setDrawingRegion(!drawingRegion)}
className={`text-xs px-2 py-0.5 rounded ${drawingRegion ? 'bg-indigo-600 text-white' : 'bg-gray-200 dark:bg-gray-700 text-gray-600 dark:text-gray-400'}`}
>
{drawingRegion ? 'Region zeichnen...' : '+ Region'}
</button>
</div>
<div
ref={rightPanelRef}
className="flex-1 overflow-auto"
onScroll={() => handleScroll('right')}
>
<div style={{ width: `${zoom}%`, minWidth: '100%' }}>
{/* Reconstruction container */}
<div
className="relative bg-white"
style={{
paddingBottom: `${aspect * 100}%`,
cursor: drawingRegion ? 'crosshair' : 'default',
}}
onMouseDown={handleReconMouseDown}
onMouseMove={handleReconMouseMove}
onMouseUp={handleReconMouseUp}
>
{/* Column background stripes */}
{session.columnsUsed.map((col, i) => {
const color = COL_TYPE_COLORS[col.type] || '#9ca3af'
return (
<div
key={`col-${i}`}
className="absolute top-0 bottom-0"
style={{
left: `${(col.x / session.imageWidth) * 100}%`,
width: `${(col.width / session.imageWidth) * 100}%`,
backgroundColor: color,
opacity: 0.06,
}}
/>
)
})}
{/* Row separator lines — derive from cells */}
{(() => {
const rowYs = new Set<number>()
for (const cell of session.cells) {
if (cell.col_index === 0 && cell.bbox_pct) {
rowYs.add(cell.bbox_pct.y)
}
}
return Array.from(rowYs).map((y, i) => (
<div
key={`row-${i}`}
className="absolute left-0 right-0"
style={{
top: `${y}%`,
height: '1px',
backgroundColor: 'rgba(0,0,0,0.08)',
}}
/>
))
})()}
{/* Cell texts */}
{session.cells.map(cell => {
if (!cell.bbox_pct || !cell.text) return null
const color = COL_TYPE_COLORS[cell.col_type] || '#374151'
return (
<span
key={cell.cell_id}
className="absolute text-[0.6em] leading-tight overflow-hidden"
style={{
left: `${cell.bbox_pct.x}%`,
top: `${cell.bbox_pct.y}%`,
width: `${cell.bbox_pct.w}%`,
height: `${cell.bbox_pct.h}%`,
color,
fontFamily: "'Liberation Sans', 'DejaVu Sans', sans-serif",
display: 'flex',
alignItems: 'center',
padding: '0 1px',
}}
title={`${cell.cell_id}: ${cell.text}`}
>
{cell.text}
</span>
)
})}
{/* Generated images at region positions */}
{imageRegions.map((region, i) => (
<div
key={`region-${i}`}
className="absolute border-2 border-dashed border-indigo-400"
style={{
left: `${region.bbox_pct.x}%`,
top: `${region.bbox_pct.y}%`,
width: `${region.bbox_pct.w}%`,
height: `${region.bbox_pct.h}%`,
}}
>
{region.image_b64 ? (
<img src={region.image_b64} alt={region.description} className="w-full h-full object-cover" />
) : (
<div className="w-full h-full flex items-center justify-center bg-indigo-50/50 text-indigo-400 text-[0.5em]">
{region.generating ? '...' : `Bild ${i + 1}`}
</div>
)}
</div>
))}
{/* Drawing rectangle */}
{dragStart && dragEnd && (
<div
className="absolute border-2 border-dashed border-red-500 bg-red-100/20 pointer-events-none"
style={{
left: `${Math.min(dragStart.x, dragEnd.x)}%`,
top: `${Math.min(dragStart.y, dragEnd.y)}%`,
width: `${Math.abs(dragEnd.x - dragStart.x)}%`,
height: `${Math.abs(dragEnd.y - dragStart.y)}%`,
}}
/>
)}
</div>
</div>
</div>
</div>
</div>
{/* Image regions panel */}
{imageRegions.length > 0 && (
<div className="border rounded-lg dark:border-gray-700 p-4">
<h4 className="text-sm font-medium text-gray-700 dark:text-gray-300 mb-3">
Bildbereiche ({imageRegions.length} gefunden)
</h4>
<div className="space-y-3">
{imageRegions.map((region, i) => (
<div key={i} className="flex items-start gap-3 p-3 bg-gray-50 dark:bg-gray-800 rounded-lg">
{/* Preview thumbnail */}
<div className="w-16 h-16 flex-shrink-0 border rounded dark:border-gray-600 overflow-hidden bg-white">
{region.image_b64 ? (
<img src={region.image_b64} alt="" className="w-full h-full object-cover" />
) : (
<div className="w-full h-full flex items-center justify-center text-gray-400 text-xs">
{Math.round(region.bbox_pct.w)}x{Math.round(region.bbox_pct.h)}%
</div>
)}
</div>
{/* Prompt + controls */}
<div className="flex-1 min-w-0 space-y-2">
<div className="flex items-center gap-2">
<span className="text-xs text-gray-500 dark:text-gray-400 flex-shrink-0">
Bereich {i + 1}:
</span>
<input
type="text"
value={region.prompt}
onChange={e => {
setImageRegions(prev => prev.map((r, j) =>
j === i ? { ...r, prompt: e.target.value } : r
))
}}
placeholder="Beschreibung / Prompt..."
className="flex-1 text-sm px-2 py-1 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
</div>
<div className="flex items-center gap-2">
<select
value={region.style}
onChange={e => {
setImageRegions(prev => prev.map((r, j) =>
j === i ? { ...r, style: e.target.value as ImageStyle } : r
))
}}
className="text-sm px-2 py-1 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
>
{STYLES.map(s => (
<option key={s.value} value={s.value}>{s.label}</option>
))}
</select>
<button
onClick={() => handleGenerateImage(i)}
disabled={!!region.generating || !region.prompt}
className="px-3 py-1 text-sm bg-teal-600 text-white rounded hover:bg-teal-700 disabled:opacity-50"
>
{region.generating ? 'Generiere...' : 'Generieren'}
</button>
<button
onClick={() => handleRemoveRegion(i)}
className="px-2 py-1 text-sm text-red-600 hover:bg-red-50 dark:hover:bg-red-900/20 rounded"
>
Entfernen
</button>
</div>
{region.description && region.description !== region.prompt && (
<p className="text-xs text-gray-400">{region.description}</p>
)}
</div>
</div>
))}
</div>
</div>
)}
{/* Notes and score */}
<div className="border rounded-lg dark:border-gray-700 p-4 space-y-3">
<div className="flex items-center gap-4">
<label className="text-sm font-medium text-gray-700 dark:text-gray-300">
Bewertung (1-10):
</label>
<input
type="number"
min={1}
max={10}
value={score ?? ''}
onChange={e => setScore(e.target.value ? parseInt(e.target.value) : null)}
className="w-20 text-sm px-2 py-1 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
<div className="flex gap-1">
{[1, 2, 3, 4, 5, 6, 7, 8, 9, 10].map(v => (
<button
key={v}
onClick={() => setScore(v)}
className={`w-7 h-7 text-xs rounded ${score === v ? 'bg-teal-600 text-white' : 'bg-gray-100 dark:bg-gray-700 text-gray-600 dark:text-gray-400 hover:bg-gray-200 dark:hover:bg-gray-600'}`}
>
{v}
</button>
))}
</div>
</div>
<div>
<label className="text-sm font-medium text-gray-700 dark:text-gray-300 block mb-1">
Notizen:
</label>
<textarea
value={notes}
onChange={e => setNotes(e.target.value)}
rows={3}
placeholder="Anmerkungen zur Qualitaet der Rekonstruktion..."
className="w-full text-sm px-3 py-2 border rounded dark:border-gray-600 dark:bg-gray-700 dark:text-white"
/>
</div>
</div>
{/* Actions */}
<div className="flex items-center justify-between">
<div className="text-sm text-gray-500 dark:text-gray-400">
{status === 'saved' && <span className="text-green-600 dark:text-green-400">Validierung gespeichert</span>}
{status === 'saving' && <span>Speichere...</span>}
</div>
<div className="flex items-center gap-3">
<button
onClick={handleSave}
disabled={status === 'saving'}
className="px-4 py-2 text-sm bg-gray-600 text-white rounded hover:bg-gray-700 disabled:opacity-50"
>
Speichern
</button>
<button
onClick={async () => {
await handleSave()
onNext()
}}
disabled={status === 'saving'}
className="px-4 py-2 text-sm bg-teal-600 text-white rounded hover:bg-teal-700 disabled:opacity-50"
>
Abschliessen
</button>
</div>
</div>
</div>
)

View File

@@ -1,12 +1,12 @@
# OCR Pipeline - Schrittweise Seitenrekonstruktion
**Version:** 2.0.0
**Version:** 3.0.0
**Status:** Produktiv (Schritte 18 implementiert)
**URL:** https://macmini:3002/ai/ocr-pipeline
## Uebersicht
Die OCR Pipeline zerlegt den OCR-Prozess in **8 einzelne Schritte**, um eingescannte Vokabelseiten
Die OCR Pipeline zerlegt den OCR-Prozess in **8 einzelne Schritte**, um eingescannte Seiten
aus mehrspaltig gedruckten Schulbuechern Wort fuer Wort zu rekonstruieren.
Jeder Schritt kann individuell geprueft, korrigiert und mit Ground-Truth-Daten versehen werden.
@@ -20,13 +20,94 @@ Jeder Schritt kann individuell geprueft, korrigiert und mit Ground-Truth-Daten v
| 2 | Entzerrung (Dewarp) | Buchwoelbung entzerren (Vertikalkanten-Analyse) | Implementiert |
| 3 | Spaltenerkennung | Unsichtbare Spalten finden (Projektionsprofile + Wortvalidierung) | Implementiert |
| 4 | Zeilenerkennung | Horizontale Zeilen + Kopf-/Fusszeilen-Klassifikation + Luecken-Heilung | Implementiert |
| 5 | Worterkennung | Grid aus Spalten x Zeilen, OCR pro Zelle, Post-Processing | Implementiert |
| 5 | Worterkennung | Hybrid-Grid: Breite Spalten full-page, schmale cell-crop | Implementiert |
| 6 | Korrektur | Zeichenverwirrung + regel-basierte Rechtschreibkorrektur (SSE-Stream) | Implementiert |
| 7 | Rekonstruktion | Interaktive Zellenbearbeitung auf Bildhintergrund | Implementiert |
| 7 | Rekonstruktion | Interaktive Zellenbearbeitung auf Bildhintergrund (Fabric.js) | Implementiert |
| 8 | Validierung | Ground-Truth-Vergleich und Qualitaetspruefung | Implementiert |
---
## Dokumenttyp-Erkennung und Pipeline-Pfade
### Automatische Weiche: `detect_document_type()`
Nicht jedes Dokument durchlaeuft denselben Pfad. Nach den gemeinsamen Vorverarbeitungsschritten
(Deskew, Dewarp, Binarisierung) analysiert `detect_document_type()` die Seitenstruktur
**ohne OCR** — rein ueber Projektionsprofile und Textdichte-Analyse (< 2 Sekunden).
```
detect_document_type(ocr_img, img_bgr) → DocumentTypeResult
```
#### Entscheidungslogik
```mermaid
flowchart TD
A[Bild-Input] --> B[Vertikales Projektionsprofil]
B --> C{Interne Spalten-Gaps >= 2?}
C -->|Ja| D{Zeilen-Gaps >= 5?}
D -->|Ja| E["vocab_table<br/>pipeline = cell_first<br/>confidence 0.70.95"]
D -->|Nein| F{Zeilen-Gaps >= 3?}
C -->|Nein| G{Interne Spalten-Gaps >= 1?}
G -->|Ja| F
G -->|Nein| H["full_text<br/>pipeline = full_page<br/>skip: columns, rows"]
F -->|Ja| I["generic_table<br/>pipeline = cell_first<br/>confidence 0.50.85"]
F -->|Nein| H
```
| Dokumenttyp | Spalten-Gaps | Zeilen-Gaps | Pipeline | Beispiel |
|-------------|-------------|-------------|----------|----------|
| `vocab_table` | ≥ 2 | ≥ 5 | `cell_first` | 3-spaltige Schulbuch-Vokabeltabelle |
| `generic_table` | ≥ 1 | ≥ 3 | `cell_first` | 2-spaltiges Glossar |
| `full_text` | 0 | egal | `full_page` | Fliesstext, Aufsatz, Buchseite |
### Komplett-Flussdiagramm
```
┌─────────────────────────────────────────────────────────────────────┐
│ GEMEINSAME VORVERARBEITUNG (alle Dokumente) │
│ │
│ Stage 1: Render (432 DPI, 3× Zoom) │
│ Stage 2: Deskew (Hough Lines + Ensemble) │
│ Stage 3: Dewarp (Vertikalkanten-Drift, Ensemble Shear) │
│ Stage 4: Dual-Bild (ocr_img = binarisiert, layout_img = CLAHE) │
└─────────────────────────────────────┬───────────────────────────────┘
detect_document_type()
┌─────────────────┴──────────────────┐
▼ ▼
FULL-TEXT PFAD CELL-FIRST PFAD
(pipeline='full_page') (pipeline='cell_first')
│ │
Keine Spalten/Zeilen Spaltenerkennung
analyze_layout_by_words() detect_column_geometry()
Lese-Reihenfolge _detect_sub_columns()
│ expand_narrow_columns()
│ Zeilenerkennung
│ detect_row_geometry()
│ │
│ build_cell_grid_v2()
│ │
│ ┌─────────┴──────────┐
│ ▼ ▼
│ Breite Spalten Schmale Spalten
│ (>= 15% Breite) (< 15% Breite)
│ Full-Page Words Cell-Crop OCR
│ word_lookup cell_crop_v2
│ │ │
└───────────────────────────┴────────────────────┘
Post-Processing Pipeline
(Lautschrift, Komma-Split, etc.)
Schritt 6: Korrektur (Spell)
Schritt 7: Rekonstruktion
Schritt 8: Validierung
```
---
## Architektur
```
@@ -55,28 +136,31 @@ Admin-Lehrer (Next.js) klausur-service (FastAPI :8086)
```
klausur-service/backend/
├── ocr_pipeline_api.py # FastAPI Router (alle Endpoints)
├── ocr_pipeline_session_store.py # PostgreSQL Persistence
├── cv_vocab_pipeline.py # Computer Vision + NLP Algorithmen
├── services/
│ └── cv_vocab_pipeline.py # Computer Vision + NLP Algorithmen
├── ocr_pipeline_api.py # FastAPI Router (alle Endpoints)
├── ocr_pipeline_session_store.py # PostgreSQL Persistence
├── layout_reconstruction_service.py # Fabric.js JSON + PDF/DOCX Export
└── migrations/
├── 002_ocr_pipeline_sessions.sql # Basis-Schema
├── 003_add_row_result.sql # Row-Result Spalte
└── 004_add_word_result.sql # Word-Result Spalte
├── 002_ocr_pipeline_sessions.sql # Basis-Schema
├── 003_add_row_result.sql # Row-Result Spalte
└── 004_add_word_result.sql # Word-Result Spalte
admin-lehrer/
├── app/(admin)/ai/ocr-pipeline/
│ ├── page.tsx # Haupt-Page mit Session-Management
│ └── types.ts # TypeScript Interfaces
│ ├── page.tsx # Haupt-Page mit Session-Management
│ └── types.ts # TypeScript Interfaces
└── components/ocr-pipeline/
├── PipelineStepper.tsx # Fortschritts-Stepper
├── StepDeskew.tsx # Schritt 1: Begradigung
├── StepDewarp.tsx # Schritt 2: Entzerrung
├── StepColumnDetection.tsx # Schritt 3: Spaltenerkennung
├── StepRowDetection.tsx # Schritt 4: Zeilenerkennung
├── StepWordRecognition.tsx # Schritt 5: Worterkennung
├── StepLlmReview.tsx # Schritt 6: Korrektur (SSE-Stream)
├── StepReconstruction.tsx # Schritt 7: Rekonstruktion (Canvas)
── StepGroundTruth.tsx # Schritt 8: Validierung
├── PipelineStepper.tsx # Fortschritts-Stepper
├── StepDeskew.tsx # Schritt 1: Begradigung
├── StepDewarp.tsx # Schritt 2: Entzerrung
├── StepColumnDetection.tsx # Schritt 3: Spaltenerkennung
├── StepRowDetection.tsx # Schritt 4: Zeilenerkennung
├── StepWordRecognition.tsx # Schritt 5: Worterkennung
├── StepLlmReview.tsx # Schritt 6: Korrektur (SSE-Stream)
├── StepReconstruction.tsx # Schritt 7: Rekonstruktion (Canvas)
── FabricReconstructionCanvas.tsx # Fabric.js Editor
└── StepGroundTruth.tsx # Schritt 8: Validierung
```
---
@@ -94,6 +178,7 @@ Alle Endpoints unter `/api/v1/ocr-pipeline/`.
| `GET` | `/sessions/{id}` | Session-Info mit allen Step-Results |
| `PUT` | `/sessions/{id}` | Session umbenennen |
| `DELETE` | `/sessions/{id}` | Session loeschen |
| `POST` | `/sessions/{id}/detect-type` | Dokumenttyp erkennen |
### Bilder
@@ -160,6 +245,34 @@ Alle Endpoints unter `/api/v1/ocr-pipeline/`.
| Methode | Pfad | Beschreibung |
|---------|------|--------------|
| `POST` | `/sessions/{id}/reconstruction` | Zellaenderungen speichern |
| `GET` | `/sessions/{id}/reconstruction/fabric-json` | Fabric.js Canvas-Daten |
| `GET` | `/sessions/{id}/reconstruction/export/pdf` | PDF-Export (reportlab) |
| `GET` | `/sessions/{id}/reconstruction/export/docx` | DOCX-Export (python-docx) |
| `POST` | `/sessions/{id}/reconstruction/detect-images` | Bildbereiche per VLM erkennen |
| `POST` | `/sessions/{id}/reconstruction/generate-image` | Bild per mflux generieren |
| `POST` | `/sessions/{id}/reconstruction/validate` | Validierung speichern (Step 8) |
| `GET` | `/sessions/{id}/reconstruction/validation` | Validierungsdaten abrufen |
---
## Schritt 2: Entzerrung/Dewarp (Detail)
### Algorithmus: Vertikalkanten-Drift
Die Dewarp-Erkennung misst die **vertikale Spaltenkippung** (dx/dy) statt Textzeilen-Neigung:
1. Woerter werden nach X-Position in vertikale Spaltencluster gruppiert
2. Pro Cluster: Lineare Regression `x = a*y + b``a = dx/dy = tan(shear_angle)`
3. Ensemble aus drei Methoden: Textzeilen (1.5× Gewicht), Projektionsprofil (2-Pass), Vertikalkanten
4. Qualitaetspruefung: Horizontale Projektionsvarianz vor/nach Korrektur
**Schwellenwerte:**
| Parameter | Wert | Beschreibung |
|-----------|------|--------------|
| Min. Korrekturwinkel | 0.08° | Unter 0.08° wird nicht korrigiert |
| Ensemble Min-Confidence | 0.35 | Mindest-Konfidenz fuer Korrektur |
| Quality-Gate Skip | < 0.5° | Kleine Korrekturen ueberspringen Quality-Gate |
---
@@ -180,6 +293,38 @@ Bild → Binarisierung → Vertikalprofil → Lueckenerkennung → Wort-Validier
- **Phantom-Spalten-Filter (Step 9):** Spalten mit Breite < 3 % der Content-Breite UND < 3 Woerter werden als Artefakte entfernt; die angrenzenden Spalten schliessen die Luecke.
- **Spaltenzuweisung:** Woerter werden anhand des groessten horizontalen Ueberlappungsbereichs einer Spalte zugeordnet.
### Sub-Spalten-Erkennung: `_detect_sub_columns()`
Erkennt versteckte Sub-Spalten innerhalb breiter Spalten (z.B. Seitenzahl-Spalte links neben EN-Vokabeln).
**Algorithmus (Left-Edge Alignment Clustering):**
1. Fuer jede Spalte mit `width_ratio >= 0.15` und `word_count >= 5`:
2. Left-Edges aller Woerter mit `conf >= 30` sammeln
3. In Alignment-Bins clustern (8px Toleranz)
4. Linkester Bin mit >= 10% der Woerter = wahrer Spaltenanfang
5. Woerter links davon = Sub-Spalte, wenn >= 2 und < 35% Anteil
6. Neue ColumnGeometry-Objekte mit korrekten Indizes erzeugen
**Koordinatensystem:** Word `left`-Werte sind relativ zum Content-ROI (`left_x`), `ColumnGeometry.x` ist absolut. `left_x` wird als Parameter durchgereicht.
### Spalten-Erweiterung: `expand_narrow_columns()`
Laeuft **nach** `_detect_sub_columns()`. Erweitert sehr schmale Spalten (< 10% Content-Breite,
z.B. `page_ref`, `marker`) in den Weissraum zum Nachbar-Spalte hinein, aber nie ueber die
naechsten Woerter im Nachbarn hinaus (4px Sicherheitsabstand).
### Spaltentyp-Klassifikation: `classify_column_types()`
| Spaltentyp | Beschreibung | Erkennung |
|------------|--------------|-----------|
| `column_en` | Englische Vokabeln | EN-Funktionswoerter (the, a, is...) |
| `column_de` | Deutsche Uebersetzung | DE-Funktionswoerter (der, die, das...) |
| `column_example` | Beispielsaetze | Abkuerzungen, Grammatik-Marker |
| `page_ref` | Seitenzahlen | Schmal (< 20% Breite), wenige Woerter |
| `column_marker` | Dekorative Markierungen | Sehr schmal, spezielle Zeichen |
| `column_text` | Generischer Text | Fallback |
### Konfigurierbare Parameter
```python
@@ -219,29 +364,95 @@ def _heal_row_gaps(rows, top_bound, bottom_bound):
---
## Schritt 5: Worterkennung (Detail)
## Schritt 5: Worterkennung — Hybrid-Grid (Detail)
### Algorithmus: `build_cell_grid()`
### Algorithmus: `build_cell_grid_v2()`
Schritt 5 nutzt die Ergebnisse von Schritt 3 (Spalten) und Schritt 4 (Zeilen), um ein Grid
zu erstellen und jede Zelle per OCR auszulesen.
Schritt 5 nutzt eine **Hybrid-Strategie**: Breite Spalten verwenden die Full-Page-Tesseract-Woerter,
schmale Spalten werden isoliert per Cell-Crop OCR verarbeitet.
```
Spalten (Step 3): column_en | column_de | column_example
───────────┼─────────────┼────────────────
Zeilen (Step 4): R0 │ hello │ hallo │ Hello, World!
R1 │ world │ Welt │ The whole world
R2 │ book │ Buch │ Read a book
───────────┼─────────────┼────────────────
!!! success "Warum Hybrid?"
Full-Page OCR liefert gute Ergebnisse fuer breite Spalten (Saetze, IPA-Klammern, Interpunktion).
Aber bei schmalen Spalten (Seitenzahlen, Marker) „bluten" Woerter aus Nachbar-Spalten ein.
Cell-Crop isoliert jede Zelle und verhindert dieses Bleeding.
### Broad vs. Narrow — Die 15%-Schwelle
```python
_NARROW_COL_THRESHOLD_PCT = 15.0 # cv_vocab_pipeline.py
```
**Ablauf:**
| Eigenschaft | Breite Spalten (>= 15%) | Schmale Spalten (< 15%) |
|-------------|------------------------|------------------------|
| **OCR-Quelle** | Full-Page Tesseract (vorher gelaufen) | Isolierter Cell-Crop |
| **Wort-Zuweisung** | `_assign_row_words_to_columns()` | Direktes Zell-OCR |
| **Confidence-Filter** | `conf >= 30` | `conf >= 30` |
| **Text-Bereinigung** | `_clean_cell_text()` (mittel) | `_clean_cell_text_lite()` (aggressiv) |
| **Neighbour-Bleeding** | Risiko vorhanden | Verhindert (isoliert) |
| **Parallelisierung** | Sequentiell | Parallel (`max_workers=4`) |
| **OCR-Engine Label** | `word_lookup` | `cell_crop_v2` |
| **Typische Spalten** | EN-Vokabeln, DE-Uebersetzung, Beispielsaetze | Seitenzahlen, Marker |
1. **Initialer Scan:** Ganzes Bild einmal per Tesseract/RapidOCR → alle Wort-Bboxes
2. **Zuweisung:** Jedes Wort der Spalte mit groesstem horizontalem Ueberlapp zuordnen
3. **Zell-OCR Fallback:** Leere Zellen bekommen eigenen Crop + erneuten OCR-Aufruf (PSM 6/7)
4. **Batch-Spalten-OCR:** Bei vielen leeren Zellen in einer Spalte: gesamte Spalte einmal OCR-en
5. **Post-Processing:** Continuation-Rows zusammenfuehren, Lautschrift erkennen, Komma-Eintraege splitten
**Empirische Grundlage:** Typische breite Spalten liegen bei 2040% Bildbreite,
typische schmale bei 312%. Die 15%-Grenze trennt diese Gruppen sauber.
!!! note "Offener Punkt: Schwellen-Validierung"
Die 15%-Schwelle wurde an Vokabeltabellen mit 35 Spalten validiert.
Fuer eine breitere Validierung werden diverse Schulbuchseiten mit unterschiedlichen
Layouts (2-, 3-, 4-, 5-spaltig, verschiedene Verlage) benoetigt. Aktuell gibt es
in der Datenbank nur Sessions mit demselben Arbeitsblatt-Typ.
### Cell-Crop OCR: `_ocr_cell_crop()`
Isolierte OCR einer einzelnen Zelle (Spalte × Zeile Schnittflaeche):
1. **Crop:** Exakte Spalten- × Zeilengrenzen mit 3px internem Padding
2. **Density-Check:** Ueberspringe leere Zellen (`dark_ratio < 0.005`)
3. **Upscaling:** Kleine Crops (Hoehe < 80px) werden 3× vergroessert
4. **OCR:** Engine-spezifisch (Tesseract, TrOCR, RapidOCR, LightON)
5. **Fallback:** Bei leerem Ergebnis → PSM 7 (Einzelzeile) statt PSM 6
6. **Bereinigung:** `_clean_cell_text_lite()` (aggressives Noise-Filtering)
### Ablauf von `build_cell_grid_v2()`
```
Eingabe: ocr_img, column_regions, row_geometries
┌───────────┴───────────┐
│ Filter │
│ • Phantom-Zeilen │
│ • Artefakt-Zeilen │
│ • Irrelevante Spalten │
│ (header, footer, │
│ margin, ignore) │
└───────────┬───────────┘
┌───────────┴───────────┐
│ Klassifizierung │
│ Spalte.width / img_w │
│ >= 15% → broad │
│ < 15% → narrow │
└───────────┬───────────┘
┌───────────┴────────────────┐
│ │
Phase 1: Broad Phase 2: Narrow
(sequentiell) (parallel, max_workers=4)
│ │
Pro (row, col): Pro (row, col):
1. Words aus Full-Page 1. _ocr_cell_crop()
2. Filter conf >= 30 2. Isoliertes Zell-Bild
3. _words_to_reading_order 3. Upscale wenn noetig
4. _clean_cell_text() 4. _clean_cell_text_lite()
│ │
└───────────┬────────────────┘
Merge + Sortierung
(row_index, col_index)
Leere Zeilen entfernen
Ausgabe: cells[], columns_meta[]
```
### Post-Processing Pipeline (in `build_vocab_pipeline_streaming`)
@@ -264,7 +475,7 @@ Zeilen (Step 4): R0 │ hello │ hallo │ Hello, World!
### Korrektur-Engine
Schritt 6 kombiniert zwei Korrektur-Stufen, beide als SSE-Stream:
Schritt 6 kombiniert drei Korrektur-Stufen, alle als SSE-Stream:
**Stufe 1 — Zeichenverwirrungskorrektur** (`_fix_character_confusion`):
@@ -288,8 +499,9 @@ _SPELL_SUBS = {
}
```
Logik: Kandidaten werden durch Woerterbuch-Lookup validiert. Strukturregel: Verdaechtiges
Zeichen an Position 0 + Rest klein → erstes Substitut (z.B. `8en``Ben`).
**Stufe 3 — Seitenzahl-Korrektur** (`page_ref`-Felder):
Korrigiert haeufige OCR-Fehler in Seitenverweisen (z.B. `p.5g``p.59`).
### Umgebungsvariablen
@@ -318,7 +530,11 @@ Change-Format:
## Schritt 7: Rekonstruktion (Detail)
Interaktiver Canvas-Editor: Das entzerrte Originalbild wird mit 30 % Opazitaet als Hintergrund
Zwei Modi verfuegbar:
### Einfacher Modus
Das entzerrte Originalbild wird mit 30 % Opazitaet als Hintergrund
angezeigt, alle Grid-Zellen (auch leere!) werden als editierbare Textfelder darueber gelegt.
**Features:**
@@ -331,6 +547,14 @@ angezeigt, alle Grid-Zellen (auch leere!) werden als editierbare Textfelder daru
- Zoom 50200 %
- Per-Zell-Reset-Button bei geaenderten Zellen
### Fabric.js Editor
Erweiterter Canvas-Editor (`FabricReconstructionCanvas.tsx`):
- Drag & Drop fuer Zellen
- Freie Positionierung auf dem Canvas
- Export als PDF (reportlab) oder DOCX (python-docx)
```
POST /sessions/{id}/reconstruction
Body: {"cells": [{"cell_id": "r5_c2", "text": "corrected text"}]}
@@ -338,6 +562,19 @@ Body: {"cells": [{"cell_id": "r5_c2", "text": "corrected text"}]}
---
## Wichtige Konstanten
| Konstante | Wert | Datei | Beschreibung |
|-----------|------|-------|--------------|
| `_NARROW_COL_THRESHOLD_PCT` | 15.0% | cv_vocab_pipeline.py | Schwelle breit/schmal fuer Hybrid-OCR |
| `_NARROW_THRESHOLD_PCT` | 10.0% | cv_vocab_pipeline.py | Schwelle fuer Spalten-Erweiterung |
| `_MIN_WORD_CONF` | 30 | cv_vocab_pipeline.py | Mindest-Confidence fuer OCR-Woerter |
| `_PAD` | 3px | cv_vocab_pipeline.py | Internes Padding bei Cell-Crop |
| `PDF_ZOOM` | 3.0 | cv_vocab_pipeline.py | PDF-Rendering (= 432 DPI) |
| `_MIN_WORD_MARGIN` | 4px | cv_vocab_pipeline.py | Sicherheitsabstand bei Spalten-Erweiterung |
---
## Datenbank-Schema
```sql
@@ -348,6 +585,10 @@ CREATE TABLE ocr_pipeline_sessions (
status VARCHAR(50) DEFAULT 'active',
current_step INT DEFAULT 1,
-- Dokumenttyp-Erkennung
doc_type VARCHAR(50), -- 'vocab_table', 'generic_table', 'full_text'
doc_type_result JSONB, -- Vollstaendiges DetectionResult
-- Bilder (BYTEA)
original_png BYTEA,
deskewed_png BYTEA,
@@ -374,7 +615,7 @@ CREATE TABLE ocr_pipeline_sessions (
```json
{
"vocab_entries": [...],
"cells": [{"cell_id": "r0_c0", "text": "hello", "bbox_pct": {...}, ...}],
"cells": [{"cell_id": "r0_c0", "text": "hello", "bbox_pct": {...}, "ocr_engine": "word_lookup", ...}],
"columns_used": [...],
"llm_review": {
"changes": [{"row_index": 5, "field": "english", "old": "...", "new": "..."}],
@@ -399,10 +640,13 @@ CREATE TABLE ocr_pipeline_sessions (
| `onnxruntime` | latest | MIT | ONNX-Inferenz fuer RapidOCR |
| `pyspellchecker` | ≥0.8.1 | MIT | Regel-basierte OCR-Korrektur (Schritt 6) |
| `eng-to-ipa` | latest | MIT | IPA-Lautschrift-Lookup (Schritt 5) |
| `reportlab` | latest | BSD | PDF-Export (Schritt 7) |
| `python-docx` | ≥1.1.0 | MIT | DOCX-Export (Schritt 7) |
| `fabric` (JS) | ^6 | MIT | Canvas-Editor (Frontend) |
!!! info "pyspellchecker (neu seit 2026-03)"
`pyspellchecker` (MIT-Lizenz) ersetzt die LLM-basierte Korrektur als Standard-Engine.
EN+DE-Woerterbuch, ~134k Woerter. Kein Ollama notig.
EN+DE-Woerterbuch, ~134k Woerter. Kein Ollama noetig.
Umschaltbar via `REVIEW_ENGINE=llm` fuer den LLM-Pfad.
---
@@ -413,8 +657,10 @@ CREATE TABLE ocr_pipeline_sessions (
|---------|---------|------------|
| Schraeg gedruckte Seiten | Deskew erkennt Text-Rotation, nicht Seiten-Rotation | Manueller Winkel |
| Sehr kleine Schrift (< 8pt) | Tesseract PSM 7 braucht min. Zeichengroesse | Vorher zoomen |
| Handgeschriebene Eintraege | Tesseract/RapidOCR sind fuer Druckschrift optimiert | TrOCR-Engine (geplant) |
| Handgeschriebene Eintraege | Tesseract/RapidOCR sind fuer Druckschrift optimiert | TrOCR-Engine |
| Mehr als 4 Spalten | Projektionsprofil kann verschmelzen | Manuelle Spalten |
| Farbige Marker (rot/blau) | HSV-Erkennung erzeugt False Positives | Manuell im Rekonstruktions-Editor |
| 15%-Schwelle nicht breit validiert | Nur an einem Arbeitsblatt-Typ getestet | Diverse Schulbuchseiten testen |
---
@@ -425,17 +671,15 @@ CREATE TABLE ocr_pipeline_sessions (
git push origin main
# 2. Mac Mini pull + build
ssh macmini "cd /Users/benjaminadmin/Projekte/breakpilot-lehrer && git pull --no-rebase origin main"
ssh macmini "git -C /Users/benjaminadmin/Projekte/breakpilot-lehrer pull --no-rebase origin main"
# klausur-service (Backend) — bei requirements.txt Aenderungen: klausur-base neu bauen
ssh macmini "cd /Users/benjaminadmin/Projekte/breakpilot-lehrer && \
/usr/local/bin/docker compose build klausur-service && \
/usr/local/bin/docker compose up -d klausur-service"
# klausur-service (Backend)
ssh macmini "/usr/local/bin/docker compose -f /Users/benjaminadmin/Projekte/breakpilot-lehrer/docker-compose.yml build klausur-service"
ssh macmini "/usr/local/bin/docker compose -f /Users/benjaminadmin/Projekte/breakpilot-lehrer/docker-compose.yml up -d klausur-service"
# admin-lehrer (Frontend)
ssh macmini "cd /Users/benjaminadmin/Projekte/breakpilot-lehrer && \
/usr/local/bin/docker compose build admin-lehrer && \
/usr/local/bin/docker compose up -d admin-lehrer"
ssh macmini "/usr/local/bin/docker compose -f /Users/benjaminadmin/Projekte/breakpilot-lehrer/docker-compose.yml build admin-lehrer"
ssh macmini "/usr/local/bin/docker compose -f /Users/benjaminadmin/Projekte/breakpilot-lehrer/docker-compose.yml up -d admin-lehrer"
# 3. Testen unter:
# https://macmini:3002/ai/ocr-pipeline
@@ -445,9 +689,8 @@ ssh macmini "cd /Users/benjaminadmin/Projekte/breakpilot-lehrer && \
Wenn `requirements.txt` geaendert wird (z.B. neues Paket hinzugefuegt), muss zuerst
das Base-Image neu gebaut werden:
```bash
ssh macmini "cd ~/Projekte/breakpilot-lehrer && \
/usr/local/bin/docker build -f klausur-service/Dockerfile.base \
-t klausur-base:latest klausur-service/"
ssh macmini "/usr/local/bin/docker build -f /Users/benjaminadmin/Projekte/breakpilot-lehrer/klausur-service/Dockerfile.base \
-t klausur-base:latest /Users/benjaminadmin/Projekte/breakpilot-lehrer/klausur-service/"
```
---
@@ -456,6 +699,9 @@ ssh macmini "cd /Users/benjaminadmin/Projekte/breakpilot-lehrer && \
| Datum | Version | Aenderung |
|-------|---------|----------|
| 2026-03-05 | 3.0.0 | Doku-Update: Dokumenttyp-Erkennung, Hybrid-Grid, Sub-Column-Detection, Pipeline-Pfade |
| 2026-03-04 | 2.2.0 | Dewarp: Vertikalkanten-Drift statt Textzeilen-Neigung, Schwellenwerte gesenkt |
| 2026-03-04 | 2.1.0 | Sub-Column-Detection, expand_narrow_columns, Fabric.js Editor, PDF/DOCX-Export |
| 2026-03-03 | 2.0.0 | Schritte 67 implementiert; Spell-Checker, Rekonstruktions-Canvas |
| 2026-03-03 | 1.5.0 | Spaltenerkennung: volle Bildbreite fuer initialen Scan, Phantom-Filter |
| 2026-03-03 | 1.4.0 | Zeilenerkennung: Artefakt-Zeilen entfernen + Luecken-Heilung |

View File

@@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
tesseract-ocr-eng \
libgl1 \
libglib2.0-0 \
fonts-liberation \
&& rm -rf /var/lib/apt/lists/*
# Python dependencies

View File

@@ -2238,6 +2238,271 @@ async def export_reconstruction_docx(session_id: str):
raise HTTPException(status_code=501, detail="python-docx not installed")
# ---------------------------------------------------------------------------
# Step 8: Validation — Original vs. Reconstruction
# ---------------------------------------------------------------------------
STYLE_SUFFIXES = {
"educational": "educational illustration, textbook style, clear, colorful",
"cartoon": "cartoon, child-friendly, simple shapes",
"sketch": "pencil sketch, hand-drawn, black and white",
"clipart": "clipart, flat vector style, simple",
"realistic": "photorealistic, high detail",
}
class ValidationRequest(BaseModel):
notes: Optional[str] = None
score: Optional[int] = None
class GenerateImageRequest(BaseModel):
region_index: int
prompt: str
style: str = "educational"
@router.post("/sessions/{session_id}/reconstruction/detect-images")
async def detect_image_regions(session_id: str):
"""Detect illustration/image regions in the original scan using VLM.
Sends the original image to qwen2.5vl to find non-text, non-table
image areas, returning bounding boxes (in %) and descriptions.
"""
import base64
import httpx
import re
session = await get_session_db(session_id)
if not session:
raise HTTPException(status_code=404, detail=f"Session {session_id} not found")
# Get original image bytes
original_png = await get_session_image(session_id, "original")
if not original_png:
raise HTTPException(status_code=400, detail="No original image found")
# Build context from vocab entries for richer descriptions
word_result = session.get("word_result") or {}
entries = word_result.get("vocab_entries") or word_result.get("entries") or []
vocab_context = ""
if entries:
sample = entries[:10]
words = [f"{e.get('english', '')} / {e.get('german', '')}" for e in sample if e.get('english')]
if words:
vocab_context = f"\nContext: This is a vocabulary page with words like: {', '.join(words)}"
ollama_base = os.getenv("OLLAMA_BASE_URL", "http://host.docker.internal:11434")
model = os.getenv("OLLAMA_HTR_MODEL", "qwen2.5vl:32b")
prompt = (
"Analyze this scanned page. Find ALL illustration/image/picture regions "
"(NOT text, NOT table cells, NOT blank areas). "
"For each image region found, return its bounding box as percentage of page dimensions "
"and a short English description of what the image shows. "
"Reply with ONLY a JSON array like: "
'[{"x": 10, "y": 20, "w": 30, "h": 25, "description": "drawing of a cat"}] '
"where x, y, w, h are percentages (0-100) of the page width/height. "
"If there are NO images on the page, return an empty array: []"
f"{vocab_context}"
)
img_b64 = base64.b64encode(original_png).decode("utf-8")
payload = {
"model": model,
"prompt": prompt,
"images": [img_b64],
"stream": False,
}
try:
async with httpx.AsyncClient(timeout=120.0) as client:
resp = await client.post(f"{ollama_base}/api/generate", json=payload)
resp.raise_for_status()
text = resp.json().get("response", "")
# Parse JSON array from response
match = re.search(r'\[.*?\]', text, re.DOTALL)
if match:
raw_regions = json.loads(match.group(0))
else:
raw_regions = []
# Normalize to ImageRegion format
regions = []
for r in raw_regions:
regions.append({
"bbox_pct": {
"x": max(0, min(100, float(r.get("x", 0)))),
"y": max(0, min(100, float(r.get("y", 0)))),
"w": max(1, min(100, float(r.get("w", 10)))),
"h": max(1, min(100, float(r.get("h", 10)))),
},
"description": r.get("description", ""),
"prompt": r.get("description", ""),
"image_b64": None,
"style": "educational",
})
# Enrich prompts with nearby vocab context
if entries:
for region in regions:
ry = region["bbox_pct"]["y"]
rh = region["bbox_pct"]["h"]
nearby = [
e for e in entries
if e.get("bbox") and abs(e["bbox"].get("y", 0) - ry) < rh + 10
]
if nearby:
en_words = [e.get("english", "") for e in nearby if e.get("english")]
de_words = [e.get("german", "") for e in nearby if e.get("german")]
if en_words or de_words:
context = f" (vocabulary context: {', '.join(en_words[:5])}"
if de_words:
context += f" / {', '.join(de_words[:5])}"
context += ")"
region["prompt"] = region["description"] + context
# Save to ground_truth JSONB
ground_truth = session.get("ground_truth") or {}
validation = ground_truth.get("validation") or {}
validation["image_regions"] = regions
validation["detected_at"] = datetime.utcnow().isoformat()
ground_truth["validation"] = validation
await update_session_db(session_id, ground_truth=ground_truth)
if session_id in _cache:
_cache[session_id]["ground_truth"] = ground_truth
logger.info(f"Detected {len(regions)} image regions for session {session_id}")
return {"regions": regions, "count": len(regions)}
except httpx.ConnectError:
logger.warning(f"VLM not available at {ollama_base} for image detection")
return {"regions": [], "count": 0, "error": "VLM not available"}
except Exception as e:
logger.error(f"Image detection failed for {session_id}: {e}")
return {"regions": [], "count": 0, "error": str(e)}
@router.post("/sessions/{session_id}/reconstruction/generate-image")
async def generate_image_for_region(session_id: str, req: GenerateImageRequest):
"""Generate a replacement image for a detected region using mflux.
Sends the prompt (with style suffix) to the mflux-service running
natively on the Mac Mini (Metal GPU required).
"""
import httpx
session = await get_session_db(session_id)
if not session:
raise HTTPException(status_code=404, detail=f"Session {session_id} not found")
ground_truth = session.get("ground_truth") or {}
validation = ground_truth.get("validation") or {}
regions = validation.get("image_regions") or []
if req.region_index < 0 or req.region_index >= len(regions):
raise HTTPException(status_code=400, detail=f"Invalid region_index {req.region_index}, have {len(regions)} regions")
mflux_url = os.getenv("MFLUX_URL", "http://host.docker.internal:8095")
style_suffix = STYLE_SUFFIXES.get(req.style, STYLE_SUFFIXES["educational"])
full_prompt = f"{req.prompt}, {style_suffix}"
# Determine image size from region aspect ratio (snap to multiples of 64)
region = regions[req.region_index]
bbox = region["bbox_pct"]
aspect = bbox["w"] / max(bbox["h"], 1)
if aspect > 1.3:
width, height = 768, 512
elif aspect < 0.7:
width, height = 512, 768
else:
width, height = 512, 512
try:
async with httpx.AsyncClient(timeout=300.0) as client:
resp = await client.post(f"{mflux_url}/generate", json={
"prompt": full_prompt,
"width": width,
"height": height,
"steps": 4,
})
resp.raise_for_status()
data = resp.json()
image_b64 = data.get("image_b64")
if not image_b64:
return {"image_b64": None, "success": False, "error": "No image returned"}
# Save to ground_truth
regions[req.region_index]["image_b64"] = image_b64
regions[req.region_index]["prompt"] = req.prompt
regions[req.region_index]["style"] = req.style
validation["image_regions"] = regions
ground_truth["validation"] = validation
await update_session_db(session_id, ground_truth=ground_truth)
if session_id in _cache:
_cache[session_id]["ground_truth"] = ground_truth
logger.info(f"Generated image for session {session_id} region {req.region_index}")
return {"image_b64": image_b64, "success": True}
except httpx.ConnectError:
logger.warning(f"mflux-service not available at {mflux_url}")
return {"image_b64": None, "success": False, "error": f"mflux-service not available at {mflux_url}"}
except Exception as e:
logger.error(f"Image generation failed for {session_id}: {e}")
return {"image_b64": None, "success": False, "error": str(e)}
@router.post("/sessions/{session_id}/reconstruction/validate")
async def save_validation(session_id: str, req: ValidationRequest):
"""Save final validation results for step 8.
Stores notes, score, and preserves any detected/generated image regions.
Sets current_step = 8 to mark pipeline as complete.
"""
session = await get_session_db(session_id)
if not session:
raise HTTPException(status_code=404, detail=f"Session {session_id} not found")
ground_truth = session.get("ground_truth") or {}
validation = ground_truth.get("validation") or {}
validation["validated_at"] = datetime.utcnow().isoformat()
validation["notes"] = req.notes
validation["score"] = req.score
ground_truth["validation"] = validation
await update_session_db(session_id, ground_truth=ground_truth, current_step=8)
if session_id in _cache:
_cache[session_id]["ground_truth"] = ground_truth
logger.info(f"Validation saved for session {session_id}: score={req.score}")
return {"session_id": session_id, "validation": validation}
@router.get("/sessions/{session_id}/reconstruction/validation")
async def get_validation(session_id: str):
"""Retrieve saved validation data for step 8."""
session = await get_session_db(session_id)
if not session:
raise HTTPException(status_code=404, detail=f"Session {session_id} not found")
ground_truth = session.get("ground_truth") or {}
validation = ground_truth.get("validation")
return {
"session_id": session_id,
"validation": validation,
"word_result": session.get("word_result"),
}
@router.post("/sessions/{session_id}/reprocess")
async def reprocess_session(session_id: str, request: Request):
"""Re-run pipeline from a specific step, clearing downstream data.

121
scripts/mflux-service.py Normal file
View File

@@ -0,0 +1,121 @@
#!/usr/bin/env python3
"""
mflux-service — Standalone FastAPI wrapper for mflux image generation.
Runs NATIVELY on Mac Mini (requires Metal GPU, not Docker).
Generates images using Flux Schnell via the mflux library.
Setup:
python3 -m venv ~/mflux-env
source ~/mflux-env/bin/activate
pip install mflux fastapi uvicorn
Run:
source ~/mflux-env/bin/activate
python scripts/mflux-service.py
Or as a background service:
nohup ~/mflux-env/bin/python scripts/mflux-service.py > /tmp/mflux-service.log 2>&1 &
License: Apache-2.0
"""
import base64
import io
import logging
import os
import time
from typing import Optional
import uvicorn
from fastapi import FastAPI
from pydantic import BaseModel
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger("mflux-service")
app = FastAPI(title="mflux Image Generation Service", version="1.0.0")
# Lazy-loaded generator
_flux = None
def _get_flux():
"""Lazy-load the Flux model on first use."""
global _flux
if _flux is None:
logger.info("Loading Flux Schnell model (first call, may download ~12 GB)...")
from mflux import Flux1
_flux = Flux1(
model_name="schnell",
quantize=8,
)
logger.info("Flux Schnell model loaded.")
return _flux
class GenerateRequest(BaseModel):
prompt: str
width: int = 512
height: int = 512
steps: int = 4
seed: Optional[int] = None
class GenerateResponse(BaseModel):
image_b64: Optional[str] = None
success: bool = True
error: Optional[str] = None
duration_ms: int = 0
@app.get("/health")
async def health():
return {"status": "ok", "model": "flux-schnell", "gpu": "metal"}
@app.post("/generate", response_model=GenerateResponse)
async def generate_image(req: GenerateRequest):
"""Generate an image from a text prompt using Flux Schnell."""
t0 = time.time()
# Validate dimensions (must be multiples of 64 for Flux)
width = max(256, min(1024, (req.width // 64) * 64))
height = max(256, min(1024, (req.height // 64) * 64))
try:
from mflux import Config
flux = _get_flux()
image = flux.generate_image(
seed=req.seed or int(time.time()) % 2**31,
prompt=req.prompt,
config=Config(
num_inference_steps=req.steps,
height=height,
width=width,
),
)
# Convert PIL image to base64
buf = io.BytesIO()
image.save(buf, format="PNG")
buf.seek(0)
img_b64 = "data:image/png;base64," + base64.b64encode(buf.read()).decode("utf-8")
duration_ms = int((time.time() - t0) * 1000)
logger.info(f"Generated {width}x{height} image in {duration_ms}ms: {req.prompt[:60]}...")
return GenerateResponse(image_b64=img_b64, success=True, duration_ms=duration_ms)
except Exception as e:
duration_ms = int((time.time() - t0) * 1000)
logger.error(f"Generation failed: {e}")
return GenerateResponse(image_b64=None, success=False, error=str(e), duration_ms=duration_ms)
if __name__ == "__main__":
port = int(os.getenv("MFLUX_PORT", "8095"))
logger.info(f"Starting mflux-service on port {port}")
uvicorn.run(app, host="0.0.0.0", port=port)