feat: OCR pipeline v2.1 – narrow column OCR, dewarp automation, Fabric.js editor
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 24s
CI / test-go-edu-search (push) Successful in 27s
CI / test-python-klausur (push) Failing after 1m50s
CI / test-python-agent-core (push) Successful in 19s
CI / test-nodejs-website (push) Successful in 15s

Proposal B: Adaptive padding, crop upscaling, PSM selection, row-strip re-OCR
for narrow columns (<15% width) – expected accuracy boost 60-70% → 85-90%.

Proposal A: New text-line straightness detector (Method D), quality gate
(rejects counterproductive corrections), 2-pass projection refinement,
higher confidence thresholds – expected manual dewarp reduction to <10%.

Proposal C: Fabric.js canvas editor with drag/drop, inline editing, undo/redo,
opacity slider, zoom, PDF/DOCX export endpoints.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-03 22:44:14 +01:00
parent 970ec1f548
commit ab3ecc7c08
7 changed files with 1105 additions and 128 deletions

View File

@@ -0,0 +1,410 @@
'use client'
import { useCallback, useEffect, useRef, useState } from 'react'
import type { GridCell } from '@/app/(admin)/ai/ocr-pipeline/types'
const KLAUSUR_API = '/klausur-api'
// Column type → colour mapping
const COL_TYPE_COLORS: Record<string, string> = {
column_en: '#3b82f6', // blue-500
column_de: '#22c55e', // green-500
column_example: '#f97316', // orange-500
column_text: '#a855f7', // purple-500
page_ref: '#06b6d4', // cyan-500
column_marker: '#6b7280', // gray-500
}
interface FabricReconstructionCanvasProps {
sessionId: string
cells: GridCell[]
onCellsChanged: (updates: { cell_id: string; text: string }[]) => void
}
// Fabric.js types (subset used here)
interface FabricCanvas {
add: (...objects: FabricObject[]) => FabricCanvas
remove: (...objects: FabricObject[]) => FabricCanvas
setBackgroundImage: (img: FabricImage, callback: () => void) => void
renderAll: () => void
getObjects: () => FabricObject[]
dispose: () => void
on: (event: string, handler: (e: FabricEvent) => void) => void
setWidth: (w: number) => void
setHeight: (h: number) => void
getActiveObject: () => FabricObject | null
discardActiveObject: () => FabricCanvas
requestRenderAll: () => void
setZoom: (z: number) => void
getZoom: () => number
}
interface FabricObject {
type?: string
left?: number
top?: number
width?: number
height?: number
text?: string
set: (props: Record<string, unknown>) => FabricObject
get: (prop: string) => unknown
data?: Record<string, unknown>
selectable?: boolean
on?: (event: string, handler: () => void) => void
setCoords?: () => void
}
interface FabricImage extends FabricObject {
width?: number
height?: number
scaleX?: number
scaleY?: number
}
interface FabricEvent {
target?: FabricObject
e?: MouseEvent
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type FabricModule = any
export function FabricReconstructionCanvas({
sessionId,
cells,
onCellsChanged,
}: FabricReconstructionCanvasProps) {
const canvasElRef = useRef<HTMLCanvasElement>(null)
const fabricRef = useRef<FabricCanvas | null>(null)
const fabricModuleRef = useRef<FabricModule>(null)
const [ready, setReady] = useState(false)
const [opacity, setOpacity] = useState(30)
const [zoom, setZoom] = useState(100)
const [selectedCell, setSelectedCell] = useState<string | null>(null)
const [error, setError] = useState('')
// Undo/Redo
const undoStackRef = useRef<{ cellId: string; oldText: string; newText: string }[]>([])
const redoStackRef = useRef<{ cellId: string; oldText: string; newText: string }[]>([])
// ---- Initialise Fabric.js ----
useEffect(() => {
let disposed = false
async function init() {
try {
const fabricModule = await import('fabric')
if (disposed) return
fabricModuleRef.current = fabricModule
const canvasEl = canvasElRef.current
if (!canvasEl) return
const canvas = new fabricModule.Canvas(canvasEl, {
selection: true,
preserveObjectStacking: true,
}) as unknown as FabricCanvas
fabricRef.current = canvas
// Load background image
const imgUrl = `${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/image/dewarped`
const bgImg = await new Promise<FabricImage>((resolve, reject) => {
fabricModule.FabricImage.fromURL(imgUrl, { crossOrigin: 'anonymous' })
.then((img: FabricImage) => resolve(img))
.catch((err: Error) => reject(err))
})
if (disposed) return
const imgW = (bgImg.width || 800) * (bgImg.scaleX || 1)
const imgH = (bgImg.height || 600) * (bgImg.scaleY || 1)
canvas.setWidth(imgW)
canvas.setHeight(imgH)
bgImg.set({ opacity: opacity / 100, selectable: false, evented: false } as Record<string, unknown>)
canvas.setBackgroundImage(bgImg, () => {
canvas.renderAll()
})
// Add cell objects
addCellObjects(canvas, fabricModule, cells, imgW, imgH)
// Listen for text changes
canvas.on('object:modified', (e: FabricEvent) => {
if (e.target?.data?.cellId) {
const cellId = e.target.data.cellId as string
const newText = (e.target.text || '') as string
onCellsChanged([{ cell_id: cellId, text: newText }])
}
})
// Selection tracking
canvas.on('selection:created', (e: FabricEvent) => {
if (e.target?.data?.cellId) setSelectedCell(e.target.data.cellId as string)
})
canvas.on('selection:updated', (e: FabricEvent) => {
if (e.target?.data?.cellId) setSelectedCell(e.target.data.cellId as string)
})
canvas.on('selection:cleared', () => setSelectedCell(null))
setReady(true)
} catch (err) {
if (!disposed) setError(err instanceof Error ? err.message : 'Fabric.js konnte nicht geladen werden')
}
}
init()
return () => {
disposed = true
fabricRef.current?.dispose()
fabricRef.current = null
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [sessionId])
function addCellObjects(
canvas: FabricCanvas,
fabricModule: FabricModule,
gridCells: GridCell[],
imgW: number,
imgH: number,
) {
for (const cell of gridCells) {
const color = COL_TYPE_COLORS[cell.col_type] || '#6b7280'
const x = (cell.bbox_pct.x / 100) * imgW
const y = (cell.bbox_pct.y / 100) * imgH
const w = (cell.bbox_pct.w / 100) * imgW
const h = (cell.bbox_pct.h / 100) * imgH
const fontSize = Math.max(8, Math.min(18, h * 0.55))
const textObj = new fabricModule.IText(cell.text || '', {
left: x,
top: y,
width: w,
fontSize,
fontFamily: 'monospace',
fill: '#000000',
backgroundColor: `${color}22`,
padding: 2,
editable: true,
selectable: true,
lockScalingFlip: true,
data: {
cellId: cell.cell_id,
colType: cell.col_type,
rowIndex: cell.row_index,
colIndex: cell.col_index,
originalText: cell.text,
},
})
// Border colour matches column type
textObj.set({
borderColor: color,
cornerColor: color,
cornerSize: 6,
transparentCorners: false,
} as Record<string, unknown>)
canvas.add(textObj)
}
canvas.renderAll()
}
// ---- Opacity slider ----
const handleOpacityChange = useCallback((val: number) => {
setOpacity(val)
const canvas = fabricRef.current
if (!canvas) return
// Update background image opacity
// Access internal property — Fabric stores bgImage on the canvas
const bgImg = (canvas as unknown as Record<string, unknown>).backgroundImage as FabricObject | null
if (bgImg) {
bgImg.set({ opacity: val / 100 })
canvas.renderAll()
}
}, [])
// ---- Zoom ----
const handleZoomChange = useCallback((val: number) => {
setZoom(val)
const canvas = fabricRef.current
if (!canvas) return
canvas.setZoom(val / 100)
canvas.renderAll()
}, [])
// ---- Undo / Redo via keyboard ----
useEffect(() => {
const handler = (e: KeyboardEvent) => {
if (!(e.metaKey || e.ctrlKey) || e.key !== 'z') return
e.preventDefault()
const canvas = fabricRef.current
if (!canvas) return
if (e.shiftKey) {
// Redo
const action = redoStackRef.current.pop()
if (!action) return
undoStackRef.current.push(action)
const obj = canvas.getObjects().find(
(o: FabricObject) => o.data?.cellId === action.cellId
)
if (obj) {
obj.set({ text: action.newText } as Record<string, unknown>)
canvas.renderAll()
onCellsChanged([{ cell_id: action.cellId, text: action.newText }])
}
} else {
// Undo
const action = undoStackRef.current.pop()
if (!action) return
redoStackRef.current.push(action)
const obj = canvas.getObjects().find(
(o: FabricObject) => o.data?.cellId === action.cellId
)
if (obj) {
obj.set({ text: action.oldText } as Record<string, unknown>)
canvas.renderAll()
onCellsChanged([{ cell_id: action.cellId, text: action.oldText }])
}
}
}
document.addEventListener('keydown', handler)
return () => document.removeEventListener('keydown', handler)
}, [onCellsChanged])
// ---- Delete selected cell (via context-menu or Delete key) ----
useEffect(() => {
const handler = (e: KeyboardEvent) => {
if (e.key !== 'Delete' && e.key !== 'Backspace') return
// Only delete if not currently editing text inside an IText
const canvas = fabricRef.current
if (!canvas) return
const active = canvas.getActiveObject()
if (!active) return
// If the IText is in editing mode, let the keypress pass through
if ((active as unknown as Record<string, boolean>).isEditing) return
e.preventDefault()
canvas.remove(active)
canvas.discardActiveObject()
canvas.renderAll()
}
document.addEventListener('keydown', handler)
return () => document.removeEventListener('keydown', handler)
}, [])
// ---- Export helpers ----
const handleExportPdf = useCallback(() => {
window.open(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/export/pdf`,
'_blank'
)
}, [sessionId])
const handleExportDocx = useCallback(() => {
window.open(
`${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/reconstruction/export/docx`,
'_blank'
)
}, [sessionId])
if (error) {
return (
<div className="flex flex-col items-center justify-center py-8 text-red-500 text-sm">
<p>Fabric.js Editor konnte nicht geladen werden:</p>
<p className="text-xs mt-1 text-gray-400">{error}</p>
</div>
)
}
return (
<div className="space-y-2">
{/* Toolbar */}
<div className="flex items-center gap-3 bg-white dark:bg-gray-800 rounded-lg border border-gray-200 dark:border-gray-700 px-3 py-2 text-xs">
{/* Opacity slider */}
<label className="flex items-center gap-1.5 text-gray-500">
Hintergrund
<input
type="range"
min={0} max={100}
value={opacity}
onChange={e => handleOpacityChange(Number(e.target.value))}
className="w-20 h-1 accent-teal-500"
/>
<span className="w-8 text-right">{opacity}%</span>
</label>
<div className="w-px h-5 bg-gray-300 dark:bg-gray-600" />
{/* Zoom */}
<label className="flex items-center gap-1.5 text-gray-500">
Zoom
<button onClick={() => handleZoomChange(Math.max(25, zoom - 25))}
className="px-1.5 py-0.5 border border-gray-300 dark:border-gray-600 rounded hover:bg-gray-50 dark:hover:bg-gray-700">
&minus;
</button>
<span className="w-8 text-center">{zoom}%</span>
<button onClick={() => handleZoomChange(Math.min(200, zoom + 25))}
className="px-1.5 py-0.5 border border-gray-300 dark:border-gray-600 rounded hover:bg-gray-50 dark:hover:bg-gray-700">
+
</button>
<button onClick={() => handleZoomChange(100)}
className="px-1.5 py-0.5 border border-gray-300 dark:border-gray-600 rounded hover:bg-gray-50 dark:hover:bg-gray-700">
Fit
</button>
</label>
<div className="w-px h-5 bg-gray-300 dark:bg-gray-600" />
{/* Selected cell info */}
{selectedCell && (
<span className="text-gray-400">
Zelle: <span className="text-gray-600 dark:text-gray-300">{selectedCell}</span>
</span>
)}
<div className="flex-1" />
{/* Export buttons */}
<button onClick={handleExportPdf}
className="px-2.5 py-1 border border-gray-300 dark:border-gray-600 rounded hover:bg-gray-50 dark:hover:bg-gray-700">
PDF
</button>
<button onClick={handleExportDocx}
className="px-2.5 py-1 border border-gray-300 dark:border-gray-600 rounded hover:bg-gray-50 dark:hover:bg-gray-700">
DOCX
</button>
</div>
{/* Canvas */}
<div className="border rounded-lg overflow-auto dark:border-gray-700 bg-gray-100 dark:bg-gray-900"
style={{ maxHeight: '75vh' }}>
{!ready && (
<div className="flex items-center justify-center py-12">
<div className="animate-spin rounded-full h-5 w-5 border-b-2 border-teal-500" />
<span className="ml-2 text-sm text-gray-500">Canvas wird geladen...</span>
</div>
)}
<canvas ref={canvasElRef} />
</div>
{/* Legend */}
<div className="flex items-center gap-4 text-xs text-gray-500">
{Object.entries(COL_TYPE_COLORS).map(([type, color]) => (
<span key={type} className="flex items-center gap-1">
<span className="w-3 h-3 rounded" style={{ backgroundColor: color + '44', border: `1px solid ${color}` }} />
{type.replace('column_', '').replace('page_', '')}
</span>
))}
<span className="ml-auto text-gray-400">Doppelklick = Text bearbeiten | Delete = Zelle entfernen | Cmd+Z = Undo</span>
</div>
</div>
)
}

View File

@@ -1,10 +1,19 @@
'use client'
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import dynamic from 'next/dynamic'
import type { GridResult, GridCell, WordEntry } from '@/app/(admin)/ai/ocr-pipeline/types'
const KLAUSUR_API = '/klausur-api'
// Lazy-load Fabric.js canvas editor (SSR-incompatible)
const FabricReconstructionCanvas = dynamic(
() => import('./FabricReconstructionCanvas').then(m => ({ default: m.FabricReconstructionCanvas })),
{ ssr: false, loading: () => <div className="py-8 text-center text-sm text-gray-400">Editor wird geladen...</div> }
)
type EditorMode = 'simple' | 'editor'
interface StepReconstructionProps {
sessionId: string | null
onNext: () => void
@@ -26,6 +35,8 @@ export function StepReconstruction({ sessionId, onNext }: StepReconstructionProp
const [status, setStatus] = useState<'loading' | 'ready' | 'saving' | 'saved' | 'error'>('loading')
const [error, setError] = useState('')
const [cells, setCells] = useState<EditableCell[]>([])
const [gridCells, setGridCells] = useState<GridCell[]>([])
const [editorMode, setEditorMode] = useState<EditorMode>('simple')
const [editedTexts, setEditedTexts] = useState<Map<string, string>>(new Map())
const [zoom, setZoom] = useState(100)
const [imageNaturalH, setImageNaturalH] = useState(0)
@@ -70,8 +81,9 @@ export function StepReconstruction({ sessionId, onNext }: StepReconstructionProp
}
// Build editable cells from grid cells
const gridCells: GridCell[] = wordResult.cells || []
const allEditableCells: EditableCell[] = gridCells.map(c => ({
const rawGridCells: GridCell[] = wordResult.cells || []
setGridCells(rawGridCells)
const allEditableCells: EditableCell[] = rawGridCells.map(c => ({
cellId: c.cell_id,
text: c.text,
originalText: c.text,
@@ -252,6 +264,17 @@ export function StepReconstruction({ sessionId, onNext }: StepReconstructionProp
}
}, [sessionId, editedTexts, cells])
// Handler for Fabric.js editor cell changes
const handleFabricCellsChanged = useCallback((updates: { cell_id: string; text: string }[]) => {
for (const u of updates) {
setEditedTexts(prev => {
const next = new Map(prev)
next.set(u.cell_id, u.text)
return next
})
}
}, [])
const dewarpedUrl = sessionId
? `${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/image/dewarped`
: ''
@@ -332,6 +355,29 @@ export function StepReconstruction({ sessionId, onNext }: StepReconstructionProp
<h3 className="text-sm font-medium text-gray-700 dark:text-gray-300">
Schritt 7: Rekonstruktion
</h3>
{/* Mode toggle */}
<div className="flex items-center ml-2 border border-gray-300 dark:border-gray-600 rounded overflow-hidden text-xs">
<button
onClick={() => setEditorMode('simple')}
className={`px-2 py-0.5 transition-colors ${
editorMode === 'simple'
? 'bg-teal-600 text-white'
: 'hover:bg-gray-50 dark:hover:bg-gray-700 text-gray-600 dark:text-gray-400'
}`}
>
Einfach
</button>
<button
onClick={() => setEditorMode('editor')}
className={`px-2 py-0.5 transition-colors ${
editorMode === 'editor'
? 'bg-teal-600 text-white'
: 'hover:bg-gray-50 dark:hover:bg-gray-700 text-gray-600 dark:text-gray-400'
}`}
>
Editor
</button>
</div>
<span className="text-xs text-gray-400">
{cells.length} Zellen &middot; {changedCount} geaendert
{emptyCellIds.size > 0 && showEmptyHighlight && (
@@ -408,82 +454,90 @@ export function StepReconstruction({ sessionId, onNext }: StepReconstructionProp
</div>
</div>
{/* Reconstruction canvas */}
<div className="border rounded-lg overflow-auto dark:border-gray-700 bg-gray-100 dark:bg-gray-900" style={{ maxHeight: '75vh' }}>
<div
ref={containerRef}
className="relative inline-block"
style={{ transform: `scale(${zoom / 100})`, transformOrigin: 'top left' }}
>
{/* Background image at reduced opacity */}
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
ref={imageRef}
src={dewarpedUrl}
alt="Dewarped"
className="block"
style={{ opacity: 0.3 }}
onLoad={handleImageLoad}
/>
{/* Reconstruction canvas — Simple or Editor mode */}
{editorMode === 'editor' && sessionId ? (
<FabricReconstructionCanvas
sessionId={sessionId}
cells={gridCells}
onCellsChanged={handleFabricCellsChanged}
/>
) : (
<div className="border rounded-lg overflow-auto dark:border-gray-700 bg-gray-100 dark:bg-gray-900" style={{ maxHeight: '75vh' }}>
<div
ref={containerRef}
className="relative inline-block"
style={{ transform: `scale(${zoom / 100})`, transformOrigin: 'top left' }}
>
{/* Background image at reduced opacity */}
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
ref={imageRef}
src={dewarpedUrl}
alt="Dewarped"
className="block"
style={{ opacity: 0.3 }}
onLoad={handleImageLoad}
/>
{/* Empty field markers */}
{showEmptyHighlight && cells
.filter(c => emptyCellIds.has(c.cellId))
.map(cell => (
<div
key={`empty-${cell.cellId}`}
className="absolute border-2 border-dashed border-red-400/60 rounded pointer-events-none"
style={{
{/* Empty field markers */}
{showEmptyHighlight && cells
.filter(c => emptyCellIds.has(c.cellId))
.map(cell => (
<div
key={`empty-${cell.cellId}`}
className="absolute border-2 border-dashed border-red-400/60 rounded pointer-events-none"
style={{
left: `${cell.bboxPct.x}%`,
top: `${cell.bboxPct.y}%`,
width: `${cell.bboxPct.w}%`,
height: `${cell.bboxPct.h}%`,
}}
/>
))}
{/* Editable text fields at bbox positions */}
{cells.map((cell) => {
const displayText = getDisplayText(cell)
const edited = isEdited(cell)
return (
<div key={cell.cellId} className="absolute group" style={{
left: `${cell.bboxPct.x}%`,
top: `${cell.bboxPct.y}%`,
width: `${cell.bboxPct.w}%`,
height: `${cell.bboxPct.h}%`,
}}
/>
))}
{/* Editable text fields at bbox positions */}
{cells.map((cell) => {
const displayText = getDisplayText(cell)
const edited = isEdited(cell)
return (
<div key={cell.cellId} className="absolute group" style={{
left: `${cell.bboxPct.x}%`,
top: `${cell.bboxPct.y}%`,
width: `${cell.bboxPct.w}%`,
height: `${cell.bboxPct.h}%`,
}}>
<input
id={`cell-${cell.cellId}`}
type="text"
value={displayText}
onChange={(e) => handleTextChange(cell.cellId, e.target.value)}
onKeyDown={(e) => handleKeyDown(e, cell.cellId)}
className={`w-full h-full bg-transparent text-black dark:text-white border px-0.5 outline-none transition-colors ${
colTypeColor(cell.colType)
} ${edited ? 'border-green-500 bg-green-50/30 dark:bg-green-900/20' : ''}`}
style={{
fontSize: `${getFontSize(cell.bboxPct.h)}px`,
lineHeight: '1',
}}
title={`${cell.cellId} (${cell.colType})`}
/>
{/* Per-cell reset button (X) — only shown for edited cells on hover */}
{edited && (
<button
onClick={() => resetCell(cell.cellId)}
className="absolute -top-1 -right-1 w-4 h-4 bg-red-500 text-white rounded-full text-[9px] leading-none opacity-0 group-hover:opacity-100 transition-opacity flex items-center justify-center"
title="Zuruecksetzen"
>
&times;
</button>
)}
</div>
)
})}
}}>
<input
id={`cell-${cell.cellId}`}
type="text"
value={displayText}
onChange={(e) => handleTextChange(cell.cellId, e.target.value)}
onKeyDown={(e) => handleKeyDown(e, cell.cellId)}
className={`w-full h-full bg-transparent text-black dark:text-white border px-0.5 outline-none transition-colors ${
colTypeColor(cell.colType)
} ${edited ? 'border-green-500 bg-green-50/30 dark:bg-green-900/20' : ''}`}
style={{
fontSize: `${getFontSize(cell.bboxPct.h)}px`,
lineHeight: '1',
}}
title={`${cell.cellId} (${cell.colType})`}
/>
{/* Per-cell reset button (X) — only shown for edited cells on hover */}
{edited && (
<button
onClick={() => resetCell(cell.cellId)}
className="absolute -top-1 -right-1 w-4 h-4 bg-red-500 text-white rounded-full text-[9px] leading-none opacity-0 group-hover:opacity-100 transition-opacity flex items-center justify-center"
title="Zuruecksetzen"
>
&times;
</button>
)}
</div>
)
})}
</div>
</div>
</div>
)}
{/* Bottom action */}
<div className="flex justify-end">