feat: OCR word_boxes fuer pixelgenaue Overlay-Positionierung
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 37s
CI / test-go-edu-search (push) Successful in 32s
CI / test-python-klausur (push) Failing after 2m10s
CI / test-python-agent-core (push) Successful in 19s
CI / test-nodejs-website (push) Successful in 20s

Backend: _ocr_cell_crop speichert jetzt word_boxes mit exakten
Tesseract/RapidOCR Wort-Koordinaten (left, top, width, height)
im Cell-Ergebnis. Absolute Bildkoordinaten, bereits zurueckgemappt.

Frontend: Slide-Hook nutzt word_boxes direkt wenn vorhanden —
jedes Wort wird exakt an seiner OCR-Position platziert. Kein
Pixel-Scanning noetig. Fallback auf alten Slide wenn keine Boxes.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-11 19:39:49 +01:00
parent 4949863bd7
commit 0ee92e7210
4 changed files with 80 additions and 18 deletions

View File

@@ -9,16 +9,16 @@ export interface WordPosition {
}
/**
* "Slide from left" positioning algorithm.
* "Slide from left" positioning using OCR word bounding boxes.
*
* Takes ALL recognised words per cell and slides them left-to-right across
* the row's dark-pixel projection until each word "locks" onto its ink.
* If the backend provides `word_boxes` (exact per-word coordinates from
* Tesseract/RapidOCR), we place each word directly at its OCR position.
* This gives pixel-accurate overlay without any heuristic pixel scanning.
*
* Font size: fontRatio = 1.0 for all tokens (matches fallback rendering).
* Token widths: derived from canvas measureText scaled to the rendered font
* size (medianCh * 0.7), ensuring visually correct proportions.
* Fallback: if no word_boxes, slide tokens across dark-pixel projection
* (original slide algorithm).
*
* Guarantees: no words dropped, no complex matching rules needed.
* Font size: fontRatio = 1.0 for all (matches fallback rendering).
*/
export function useSlideWordPositions(
imageUrl: string,
@@ -37,6 +37,37 @@ export function useSlideWordPositions(
const imgW = img.naturalWidth
const imgH = img.naturalHeight
// Check if we can use word_boxes (fast path — no canvas needed)
const hasWordBoxes = cells.some(c => c.word_boxes && c.word_boxes.length > 0)
if (hasWordBoxes) {
// --- FAST PATH: use OCR word bounding boxes directly ---
const positions = new Map<string, WordPosition[]>()
for (const cell of cells) {
if (!cell.bbox_pct || !cell.text) continue
const boxes = cell.word_boxes
if (!boxes || boxes.length === 0) continue
const wordPos: WordPosition[] = boxes
.filter(wb => wb.text.trim())
.map(wb => ({
xPct: (wb.left / imgW) * 100,
wPct: (wb.width / imgW) * 100,
text: wb.text,
fontRatio: 1.0,
}))
if (wordPos.length > 0) {
positions.set(cell.cell_id, wordPos)
}
}
setResult(positions)
return
}
// --- SLOW PATH: pixel-projection slide (fallback if no word_boxes) ---
const canvas = document.createElement('canvas')
canvas.width = imgW
canvas.height = imgH
@@ -56,7 +87,6 @@ export function useSlideWordPositions(
const fontFam = "'Liberation Sans', Arial, sans-serif"
ctx.font = `${refFontSize}px ${fontFam}`
// --- Global font scale from median cell height ---
const cellHeights = cells
.filter(c => c.bbox_pct && c.bbox_pct.h > 0)
.map(c => Math.round(c.bbox_pct.h / 100 * imgH))
@@ -65,7 +95,6 @@ export function useSlideWordPositions(
? cellHeights[Math.floor(cellHeights.length / 2)]
: 30
// measureScale maps measureText pixels → image pixels at rendered font
const renderedFontImgPx = medianCh * 0.7
const measureScale = renderedFontImgPx / refFontSize
const spaceWidthPx = Math.max(2, Math.round(ctx.measureText(' ').width * measureScale))
@@ -91,7 +120,6 @@ export function useSlideWordPositions(
if (cy < 0) cy = 0
if (cx + cw > imgW || cy + ch > imgH) continue
// --- Dark-pixel projection ---
const imageData = ctx.getImageData(cx, cy, cw, ch)
const proj = new Float32Array(cw)
for (let y = 0; y < ch; y++) {
@@ -111,23 +139,18 @@ export function useSlideWordPositions(
ink.reverse()
}
// --- Split into individual tokens ---
const tokens = cell.text.split(/\s+/).filter(Boolean)
if (tokens.length === 0) continue
// Token widths in image pixels (at rendered font size)
const tokenWidthsPx = tokens.map(t =>
Math.max(4, Math.round(ctx.measureText(t).width * measureScale))
)
// --- Slide each token left-to-right ---
const wordPos: WordPosition[] = []
let cursor = 0
for (let ti = 0; ti < tokens.length; ti++) {
const tokenW = tokenWidthsPx[ti]
// Find first x from cursor where ≥15% of span has ink
const coverageNeeded = Math.max(1, Math.round(tokenW * 0.15))
let bestX = cursor
@@ -143,14 +166,12 @@ export function useSlideWordPositions(
bestX = x
break
}
// Safety: don't scan more than 30% of cell width past cursor
if (x > cursor + cw * 0.3 && ti > 0) {
bestX = cursor
break
}
}
// Clamp to cell bounds
if (bestX + tokenW > cw) {
bestX = Math.max(0, cw - tokenW)
}
@@ -162,7 +183,6 @@ export function useSlideWordPositions(
fontRatio: 1.0,
})
// Advance cursor: past this token + space
cursor = bestX + tokenW + spaceWidthPx
}