feat: Slide-Modus als alternative Wort-Positionierung im Overlay
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 34s
CI / test-go-edu-search (push) Successful in 33s
CI / test-python-klausur (push) Failing after 2m9s
CI / test-python-agent-core (push) Successful in 23s
CI / test-nodejs-website (push) Successful in 24s
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 34s
CI / test-go-edu-search (push) Successful in 33s
CI / test-python-klausur (push) Failing after 2m9s
CI / test-python-agent-core (push) Successful in 23s
CI / test-nodejs-website (push) Successful in 24s
Neuer Hook useSlideWordPositions: Schiebt alle erkannten Woerter von links nach rechts ueber die Pixel-Projektion bis jedes Wort auf seiner Tinte einrastet. Kein Wort geht verloren, keine Cluster-Matching-Regeln noetig. Toggle-Button (Slide/Cluster) in der Overlay-Toolbar zum Umschalten. Bestehender Cluster-Algorithmus bleibt als Alternative erhalten. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
|||||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||||
import type { GridResult, GridCell, RowResult, RowItem } from '@/app/(admin)/ai/ocr-overlay/types'
|
import type { GridResult, GridCell, RowResult, RowItem } from '@/app/(admin)/ai/ocr-overlay/types'
|
||||||
import { usePixelWordPositions } from './usePixelWordPositions'
|
import { usePixelWordPositions } from './usePixelWordPositions'
|
||||||
|
import { useSlideWordPositions } from './useSlideWordPositions'
|
||||||
|
|
||||||
const KLAUSUR_API = '/klausur-api'
|
const KLAUSUR_API = '/klausur-api'
|
||||||
|
|
||||||
@@ -42,19 +43,27 @@ export function OverlayReconstruction({ sessionId, onNext }: OverlayReconstructi
|
|||||||
const [imageRotation, setImageRotation] = useState<0 | 180>(0)
|
const [imageRotation, setImageRotation] = useState<0 | 180>(0)
|
||||||
const [textOpacity, setTextOpacity] = useState(100)
|
const [textOpacity, setTextOpacity] = useState(100)
|
||||||
const [textColor, setTextColor] = useState<'red' | 'blue' | 'black'>('red')
|
const [textColor, setTextColor] = useState<'red' | 'blue' | 'black'>('red')
|
||||||
|
const [positioningMode, setPositioningMode] = useState<'cluster' | 'slide'>('slide')
|
||||||
const reconRef = useRef<HTMLDivElement>(null)
|
const reconRef = useRef<HTMLDivElement>(null)
|
||||||
const [reconWidth, setReconWidth] = useState(0)
|
const [reconWidth, setReconWidth] = useState(0)
|
||||||
|
|
||||||
// Pixel-based word positions
|
// Pixel-based word positions (both algorithms run, toggle selects which to use)
|
||||||
const overlayImageUrl = sessionId
|
const overlayImageUrl = sessionId
|
||||||
? `${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/image/cropped`
|
? `${KLAUSUR_API}/api/v1/ocr-pipeline/sessions/${sessionId}/image/cropped`
|
||||||
: ''
|
: ''
|
||||||
const cellWordPositions = usePixelWordPositions(
|
const clusterPositions = usePixelWordPositions(
|
||||||
overlayImageUrl,
|
overlayImageUrl,
|
||||||
gridCells,
|
gridCells,
|
||||||
status === 'ready',
|
status === 'ready',
|
||||||
imageRotation,
|
imageRotation,
|
||||||
)
|
)
|
||||||
|
const slidePositions = useSlideWordPositions(
|
||||||
|
overlayImageUrl,
|
||||||
|
gridCells,
|
||||||
|
status === 'ready',
|
||||||
|
imageRotation,
|
||||||
|
)
|
||||||
|
const cellWordPositions = positioningMode === 'slide' ? slidePositions : clusterPositions
|
||||||
|
|
||||||
// Track container width
|
// Track container width
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -395,6 +404,23 @@ export function OverlayReconstruction({ sessionId, onNext }: OverlayReconstructi
|
|||||||
|
|
||||||
<div className="w-px h-5 bg-gray-300 dark:bg-gray-600 mx-1" />
|
<div className="w-px h-5 bg-gray-300 dark:bg-gray-600 mx-1" />
|
||||||
|
|
||||||
|
{/* Positioning mode toggle */}
|
||||||
|
<button
|
||||||
|
onClick={() => setPositioningMode(m => m === 'slide' ? 'cluster' : 'slide')}
|
||||||
|
className={`px-2 py-1 text-xs rounded border transition-colors ${
|
||||||
|
positioningMode === 'slide'
|
||||||
|
? 'bg-orange-500 text-white border-orange-500'
|
||||||
|
: 'bg-white dark:bg-gray-700 text-gray-600 dark:text-gray-400 border-gray-300 dark:border-gray-600'
|
||||||
|
}`}
|
||||||
|
title={positioningMode === 'slide'
|
||||||
|
? 'Slide-Modus: Woerter von links nach rechts schieben (klick fuer Cluster-Modus)'
|
||||||
|
: 'Cluster-Modus: Woerter an Pixel-Cluster zuordnen (klick fuer Slide-Modus)'}
|
||||||
|
>
|
||||||
|
{positioningMode === 'slide' ? 'Slide' : 'Cluster'}
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<div className="w-px h-5 bg-gray-300 dark:bg-gray-600 mx-1" />
|
||||||
|
|
||||||
{/* Text color */}
|
{/* Text color */}
|
||||||
{(['red', 'blue', 'black'] as const).map(c => (
|
{(['red', 'blue', 'black'] as const).map(c => (
|
||||||
<button
|
<button
|
||||||
|
|||||||
224
admin-lehrer/components/ocr-overlay/useSlideWordPositions.ts
Normal file
224
admin-lehrer/components/ocr-overlay/useSlideWordPositions.ts
Normal file
@@ -0,0 +1,224 @@
|
|||||||
|
import { useEffect, useState } from 'react'
|
||||||
|
import type { GridCell } from '@/app/(admin)/ai/ocr-overlay/types'
|
||||||
|
|
||||||
|
export interface WordPosition {
|
||||||
|
xPct: number
|
||||||
|
wPct: number
|
||||||
|
text: string
|
||||||
|
fontRatio: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Alternative positioning algorithm: "slide from left".
|
||||||
|
*
|
||||||
|
* Instead of matching text groups to pixel clusters (which can lose words),
|
||||||
|
* this algorithm takes ALL recognised words and slides them left-to-right
|
||||||
|
* across the row's dark-pixel projection until each word "locks" onto its
|
||||||
|
* ink coverage.
|
||||||
|
*
|
||||||
|
* Algorithm per cell:
|
||||||
|
* 1. Build horizontal dark-pixel projection (same as cluster approach).
|
||||||
|
* 2. Split the cell text into individual tokens (words/symbols).
|
||||||
|
* 3. Measure each token's expected pixel width (canvas measureText).
|
||||||
|
* 4. Slide a cursor from x=0 rightward. For each token, find the first
|
||||||
|
* x position where the projection has enough dark pixels under the
|
||||||
|
* token's width span (≥ coverageThreshold of the span is "inked").
|
||||||
|
* 5. Lock the token at that x, advance cursor past it + a small gap.
|
||||||
|
*
|
||||||
|
* This guarantees:
|
||||||
|
* - ALL words appear (nothing is dropped)
|
||||||
|
* - Original spacing is roughly preserved (words land on their ink)
|
||||||
|
* - Box borders/lines are naturally covered by "|" / "l" tokens
|
||||||
|
* - No complex cluster-matching or artifact-merging rules needed
|
||||||
|
*
|
||||||
|
* Returns Map<cell_id, WordPosition[]>.
|
||||||
|
*/
|
||||||
|
export function useSlideWordPositions(
|
||||||
|
imageUrl: string,
|
||||||
|
cells: GridCell[],
|
||||||
|
active: boolean,
|
||||||
|
rotation: 0 | 180 = 0,
|
||||||
|
): Map<string, WordPosition[]> {
|
||||||
|
const [result, setResult] = useState<Map<string, WordPosition[]>>(new Map())
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!active || cells.length === 0 || !imageUrl) return
|
||||||
|
|
||||||
|
const img = new Image()
|
||||||
|
img.crossOrigin = 'anonymous'
|
||||||
|
img.onload = () => {
|
||||||
|
const imgW = img.naturalWidth
|
||||||
|
const imgH = img.naturalHeight
|
||||||
|
|
||||||
|
const canvas = document.createElement('canvas')
|
||||||
|
canvas.width = imgW
|
||||||
|
canvas.height = imgH
|
||||||
|
const ctx = canvas.getContext('2d')
|
||||||
|
if (!ctx) return
|
||||||
|
|
||||||
|
if (rotation === 180) {
|
||||||
|
ctx.translate(imgW, imgH)
|
||||||
|
ctx.rotate(Math.PI)
|
||||||
|
ctx.drawImage(img, 0, 0)
|
||||||
|
ctx.setTransform(1, 0, 0, 1, 0, 0)
|
||||||
|
} else {
|
||||||
|
ctx.drawImage(img, 0, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
const refFontSize = 40
|
||||||
|
const fontFam = "'Liberation Sans', Arial, sans-serif"
|
||||||
|
ctx.font = `${refFontSize}px ${fontFam}`
|
||||||
|
|
||||||
|
const positions = new Map<string, WordPosition[]>()
|
||||||
|
|
||||||
|
for (const cell of cells) {
|
||||||
|
if (!cell.bbox_pct || !cell.text) continue
|
||||||
|
|
||||||
|
// --- Get cell rectangle in image pixels ---
|
||||||
|
let cx: number, cy: number
|
||||||
|
const cw = Math.round(cell.bbox_pct.w / 100 * imgW)
|
||||||
|
const ch = Math.round(cell.bbox_pct.h / 100 * imgH)
|
||||||
|
|
||||||
|
if (rotation === 180) {
|
||||||
|
cx = Math.round((100 - cell.bbox_pct.x - cell.bbox_pct.w) / 100 * imgW)
|
||||||
|
cy = Math.round((100 - cell.bbox_pct.y - cell.bbox_pct.h) / 100 * imgH)
|
||||||
|
} else {
|
||||||
|
cx = Math.round(cell.bbox_pct.x / 100 * imgW)
|
||||||
|
cy = Math.round(cell.bbox_pct.y / 100 * imgH)
|
||||||
|
}
|
||||||
|
if (cw <= 0 || ch <= 0) continue
|
||||||
|
if (cx < 0) cx = 0
|
||||||
|
if (cy < 0) cy = 0
|
||||||
|
if (cx + cw > imgW || cy + ch > imgH) continue
|
||||||
|
|
||||||
|
// --- Build dark-pixel projection ---
|
||||||
|
const imageData = ctx.getImageData(cx, cy, cw, ch)
|
||||||
|
const proj = new Float32Array(cw)
|
||||||
|
for (let y = 0; y < ch; y++) {
|
||||||
|
for (let x = 0; x < cw; x++) {
|
||||||
|
const idx = (y * cw + x) * 4
|
||||||
|
const lum = 0.299 * imageData.data[idx] + 0.587 * imageData.data[idx + 1] + 0.114 * imageData.data[idx + 2]
|
||||||
|
if (lum < 128) proj[x]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dark pixel threshold per column (minimum to count as "inked")
|
||||||
|
const threshold = Math.max(1, ch * 0.03)
|
||||||
|
|
||||||
|
// Build binary ink mask: true if column has enough dark pixels
|
||||||
|
const ink = new Uint8Array(cw)
|
||||||
|
for (let x = 0; x < cw; x++) {
|
||||||
|
ink[x] = proj[x] >= threshold ? 1 : 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// For 180° rotation, flip the ink mask
|
||||||
|
if (rotation === 180) {
|
||||||
|
ink.reverse()
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Split text into tokens ---
|
||||||
|
// Use triple-space groups first (preserving OCR column separation),
|
||||||
|
// then split each group into individual words for fine positioning.
|
||||||
|
const tokens = cell.text.split(/\s+/).filter(Boolean)
|
||||||
|
if (tokens.length === 0) continue
|
||||||
|
|
||||||
|
// Measure each token's width in pixels (at reference font size)
|
||||||
|
const tokenWidths = tokens.map(t => ctx.measureText(t).width)
|
||||||
|
|
||||||
|
// Total measured width of all tokens + inter-word spaces
|
||||||
|
const spaceWidth = ctx.measureText(' ').width
|
||||||
|
const totalTextW = tokenWidths.reduce((a, b) => a + b, 0) + (tokens.length - 1) * spaceWidth
|
||||||
|
|
||||||
|
// Scale factor: how much to scale reference measurements to cell pixels
|
||||||
|
// We use the total ink coverage as reference for total text width.
|
||||||
|
let totalInk = 0
|
||||||
|
for (let x = 0; x < cw; x++) totalInk += ink[x]
|
||||||
|
|
||||||
|
// If almost no ink, fall back to centering
|
||||||
|
if (totalInk < 3) continue
|
||||||
|
|
||||||
|
// The scale maps measured text width → pixel width on the image
|
||||||
|
const scale = Math.min(totalInk / totalTextW, cw / totalTextW)
|
||||||
|
|
||||||
|
// --- Slide each token from left to right ---
|
||||||
|
const wordPos: WordPosition[] = []
|
||||||
|
let cursor = 0 // current search position in cell pixels
|
||||||
|
const minGapPx = Math.max(2, Math.round(cw * 0.005)) // minimum gap between tokens
|
||||||
|
|
||||||
|
for (let ti = 0; ti < tokens.length; ti++) {
|
||||||
|
const tokenW = Math.round(tokenWidths[ti] * scale)
|
||||||
|
if (tokenW <= 0) continue
|
||||||
|
|
||||||
|
// Find first position from cursor where the token has enough ink coverage.
|
||||||
|
// "Enough" = at least 15% of the token's width has ink underneath.
|
||||||
|
const coverageNeeded = Math.max(1, Math.round(tokenW * 0.15))
|
||||||
|
let bestX = cursor
|
||||||
|
|
||||||
|
for (let x = cursor; x <= cw - tokenW; x++) {
|
||||||
|
let inkCount = 0
|
||||||
|
for (let dx = 0; dx < tokenW; dx++) {
|
||||||
|
inkCount += ink[x + dx]
|
||||||
|
}
|
||||||
|
if (inkCount >= coverageNeeded) {
|
||||||
|
bestX = x
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// If we've scanned way past where ink should be, just use cursor
|
||||||
|
if (x > cursor + cw * 0.3 && ti > 0) {
|
||||||
|
bestX = cursor
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute font size from token width vs measured width
|
||||||
|
const autoFontPx = refFontSize * (tokenW / tokenWidths[ti])
|
||||||
|
const fontRatio = Math.min(autoFontPx / ch, 1.0)
|
||||||
|
|
||||||
|
// Convert pixel position to percentage within cell, then to image %
|
||||||
|
const xInCellPct = bestX / cw
|
||||||
|
const wInCellPct = tokenW / cw
|
||||||
|
|
||||||
|
wordPos.push({
|
||||||
|
xPct: cell.bbox_pct.x + xInCellPct * cell.bbox_pct.w,
|
||||||
|
wPct: wInCellPct * cell.bbox_pct.w,
|
||||||
|
text: tokens[ti],
|
||||||
|
fontRatio,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Advance cursor past this token + gap
|
||||||
|
cursor = bestX + tokenW + minGapPx
|
||||||
|
}
|
||||||
|
|
||||||
|
if (wordPos.length > 0) {
|
||||||
|
positions.set(cell.cell_id, wordPos)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalise font: use mode fontRatio for all words
|
||||||
|
const allRatios: number[] = []
|
||||||
|
for (const wps of positions.values()) {
|
||||||
|
for (const wp of wps) allRatios.push(wp.fontRatio)
|
||||||
|
}
|
||||||
|
if (allRatios.length > 0) {
|
||||||
|
const buckets = new Map<number, number>()
|
||||||
|
for (const r of allRatios) {
|
||||||
|
const key = Math.round(r * 50) / 50
|
||||||
|
buckets.set(key, (buckets.get(key) || 0) + 1)
|
||||||
|
}
|
||||||
|
let modeRatio = allRatios[0]
|
||||||
|
let modeCount = 0
|
||||||
|
for (const [ratio, count] of buckets) {
|
||||||
|
if (count > modeCount) { modeRatio = ratio; modeCount = count }
|
||||||
|
}
|
||||||
|
for (const wps of positions.values()) {
|
||||||
|
for (const wp of wps) wp.fontRatio = modeRatio
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
setResult(positions)
|
||||||
|
}
|
||||||
|
img.src = imageUrl
|
||||||
|
}, [active, cells, imageUrl, rotation])
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user