Some checks failed
CI / test-go-edu-search (push) Has been cancelled
CI / go-lint (push) Has been cancelled
CI / python-lint (push) Has been cancelled
CI / nodejs-lint (push) Has been cancelled
CI / test-go-school (push) Has been cancelled
CI / test-python-klausur (push) Has been cancelled
CI / test-python-agent-core (push) Has been cancelled
CI / test-nodejs-website (push) Has been cancelled
Replace sequential 1:1 token-to-box mapping with fuzzy text matching. Each token from cell.text finds its best matching word_box by text similarity (normalized prefix match + substring bonus). Handles: - Reordered boxes (different sort between text and boxes) - IPA corrections changing token boundaries - Token/box count mismatches Unmatched tokens get interpolated positions from matched neighbors. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
290 lines
9.4 KiB
TypeScript
290 lines
9.4 KiB
TypeScript
import { useEffect, useState } from 'react'
|
|
import type { GridCell } from '@/app/(admin)/ai/ocr-overlay/types'
|
|
|
|
export interface WordPosition {
|
|
xPct: number
|
|
wPct: number
|
|
text: string
|
|
fontRatio: number
|
|
}
|
|
|
|
/**
|
|
* "Slide from left" positioning using OCR word bounding boxes.
|
|
*
|
|
* TEXT comes from cell.text (cleaned, IPA-corrected).
|
|
* POSITIONS come from word_boxes (exact OCR coordinates).
|
|
*
|
|
* Tokens from cell.text are matched 1:1 (in order) to word_boxes
|
|
* sorted left-to-right. This guarantees:
|
|
* - ALL words from cell.text appear (no dropping)
|
|
* - Words preserve their reading order
|
|
* - Each word lands on its correct black-text position
|
|
* - No red words overlap each other
|
|
*
|
|
* If token count != box count, extra tokens get estimated positions
|
|
* (spread across remaining space).
|
|
*
|
|
* Fallback: pixel-projection slide if no word_boxes available.
|
|
*/
|
|
export function useSlideWordPositions(
|
|
imageUrl: string,
|
|
cells: GridCell[],
|
|
active: boolean,
|
|
rotation: 0 | 180 = 0,
|
|
): Map<string, WordPosition[]> {
|
|
const [result, setResult] = useState<Map<string, WordPosition[]>>(new Map())
|
|
|
|
useEffect(() => {
|
|
if (!active || cells.length === 0 || !imageUrl) return
|
|
|
|
const img = new Image()
|
|
img.crossOrigin = 'anonymous'
|
|
img.onload = () => {
|
|
const imgW = img.naturalWidth
|
|
const imgH = img.naturalHeight
|
|
|
|
const hasWordBoxes = cells.some(c => c.word_boxes && c.word_boxes.length > 0)
|
|
|
|
if (hasWordBoxes) {
|
|
// --- WORD-BOX PATH: use OCR positions with cell.text tokens ---
|
|
// Uses fuzzy text matching to pair each token with its best box,
|
|
// handling reordering, IPA corrections, and token count mismatches.
|
|
const positions = new Map<string, WordPosition[]>()
|
|
|
|
for (const cell of cells) {
|
|
if (!cell.bbox_pct || !cell.text) continue
|
|
|
|
const tokens = cell.text.split(/\s+/).filter(Boolean)
|
|
if (tokens.length === 0) continue
|
|
|
|
const boxes = (cell.word_boxes || [])
|
|
.filter(wb => wb.text.trim())
|
|
|
|
if (boxes.length === 0) {
|
|
const fallbackW = cell.bbox_pct.w / tokens.length
|
|
const wordPos = tokens.map((t, i) => ({
|
|
xPct: cell.bbox_pct.x + i * fallbackW,
|
|
wPct: fallbackW,
|
|
text: t,
|
|
fontRatio: 1.0,
|
|
}))
|
|
positions.set(cell.cell_id, wordPos)
|
|
continue
|
|
}
|
|
|
|
// Match each token to its best box by text similarity.
|
|
// Normalize: lowercase, strip brackets/punctuation for comparison.
|
|
const norm = (s: string) => s.toLowerCase().replace(/[^a-z0-9äöüß]/g, '')
|
|
|
|
const used = new Set<number>()
|
|
const tokenBoxIdx: (number | null)[] = []
|
|
|
|
for (const token of tokens) {
|
|
const tn = norm(token)
|
|
let bestIdx = -1
|
|
let bestScore = 0
|
|
|
|
for (let bi = 0; bi < boxes.length; bi++) {
|
|
if (used.has(bi)) continue
|
|
const bn = norm(boxes[bi].text)
|
|
// Score: length of common prefix / max length
|
|
let common = 0
|
|
const minLen = Math.min(tn.length, bn.length)
|
|
for (let k = 0; k < minLen; k++) {
|
|
if (tn[k] === bn[k]) common++
|
|
else break
|
|
}
|
|
// Also check if token is a substring of box text or vice versa
|
|
const containsBonus = (bn.includes(tn) || tn.includes(bn)) ? 0.5 : 0
|
|
const score = (minLen > 0 ? common / Math.max(tn.length, bn.length) : 0) + containsBonus
|
|
if (score > bestScore) {
|
|
bestScore = score
|
|
bestIdx = bi
|
|
}
|
|
}
|
|
|
|
if (bestIdx >= 0 && bestScore > 0.2) {
|
|
used.add(bestIdx)
|
|
tokenBoxIdx.push(bestIdx)
|
|
} else {
|
|
tokenBoxIdx.push(null) // no match
|
|
}
|
|
}
|
|
|
|
// Build positions: matched tokens get box positions,
|
|
// unmatched tokens get interpolated between neighbors.
|
|
const wordPos: WordPosition[] = []
|
|
|
|
for (let ti = 0; ti < tokens.length; ti++) {
|
|
const bi = tokenBoxIdx[ti]
|
|
if (bi !== null) {
|
|
const box = boxes[bi]
|
|
wordPos.push({
|
|
xPct: (box.left / imgW) * 100,
|
|
wPct: (box.width / imgW) * 100,
|
|
text: tokens[ti],
|
|
fontRatio: 1.0,
|
|
})
|
|
} else {
|
|
// Interpolate: find nearest matched neighbor before/after
|
|
let prevPx = cell.bbox_pct.x / 100 * imgW
|
|
let prevW = 0
|
|
for (let p = ti - 1; p >= 0; p--) {
|
|
if (tokenBoxIdx[p] !== null) {
|
|
const pb = boxes[tokenBoxIdx[p]!]
|
|
prevPx = pb.left + pb.width + 5
|
|
prevW = pb.width
|
|
break
|
|
}
|
|
}
|
|
const estW = prevW > 0 ? prevW : (cell.bbox_pct.w / 100 * imgW / tokens.length)
|
|
wordPos.push({
|
|
xPct: (prevPx / imgW) * 100,
|
|
wPct: (estW / imgW) * 100,
|
|
text: tokens[ti],
|
|
fontRatio: 1.0,
|
|
})
|
|
}
|
|
}
|
|
|
|
if (wordPos.length > 0) {
|
|
positions.set(cell.cell_id, wordPos)
|
|
}
|
|
}
|
|
|
|
setResult(positions)
|
|
return
|
|
}
|
|
|
|
// --- FALLBACK: pixel-projection slide (no word_boxes) ---
|
|
const canvas = document.createElement('canvas')
|
|
canvas.width = imgW
|
|
canvas.height = imgH
|
|
const ctx = canvas.getContext('2d')
|
|
if (!ctx) return
|
|
|
|
if (rotation === 180) {
|
|
ctx.translate(imgW, imgH)
|
|
ctx.rotate(Math.PI)
|
|
ctx.drawImage(img, 0, 0)
|
|
ctx.setTransform(1, 0, 0, 1, 0, 0)
|
|
} else {
|
|
ctx.drawImage(img, 0, 0)
|
|
}
|
|
|
|
const refFontSize = 40
|
|
const fontFam = "'Liberation Sans', Arial, sans-serif"
|
|
ctx.font = `${refFontSize}px ${fontFam}`
|
|
|
|
const cellHeights = cells
|
|
.filter(c => c.bbox_pct && c.bbox_pct.h > 0)
|
|
.map(c => Math.round(c.bbox_pct.h / 100 * imgH))
|
|
.sort((a, b) => a - b)
|
|
const medianCh = cellHeights.length > 0
|
|
? cellHeights[Math.floor(cellHeights.length / 2)]
|
|
: 30
|
|
|
|
const renderedFontImgPx = medianCh * 0.7
|
|
const measureScale = renderedFontImgPx / refFontSize
|
|
const spaceWidthPx = Math.max(2, Math.round(ctx.measureText(' ').width * measureScale))
|
|
|
|
const positions = new Map<string, WordPosition[]>()
|
|
|
|
for (const cell of cells) {
|
|
if (!cell.bbox_pct || !cell.text) continue
|
|
|
|
let cx: number, cy: number
|
|
const cw = Math.round(cell.bbox_pct.w / 100 * imgW)
|
|
const ch = Math.round(cell.bbox_pct.h / 100 * imgH)
|
|
|
|
if (rotation === 180) {
|
|
cx = Math.round((100 - cell.bbox_pct.x - cell.bbox_pct.w) / 100 * imgW)
|
|
cy = Math.round((100 - cell.bbox_pct.y - cell.bbox_pct.h) / 100 * imgH)
|
|
} else {
|
|
cx = Math.round(cell.bbox_pct.x / 100 * imgW)
|
|
cy = Math.round(cell.bbox_pct.y / 100 * imgH)
|
|
}
|
|
if (cw <= 0 || ch <= 0) continue
|
|
if (cx < 0) cx = 0
|
|
if (cy < 0) cy = 0
|
|
if (cx + cw > imgW || cy + ch > imgH) continue
|
|
|
|
const imageData = ctx.getImageData(cx, cy, cw, ch)
|
|
const proj = new Float32Array(cw)
|
|
for (let y = 0; y < ch; y++) {
|
|
for (let x = 0; x < cw; x++) {
|
|
const idx = (y * cw + x) * 4
|
|
const lum = 0.299 * imageData.data[idx] + 0.587 * imageData.data[idx + 1] + 0.114 * imageData.data[idx + 2]
|
|
if (lum < 128) proj[x]++
|
|
}
|
|
}
|
|
|
|
const threshold = Math.max(1, ch * 0.03)
|
|
const ink = new Uint8Array(cw)
|
|
for (let x = 0; x < cw; x++) {
|
|
ink[x] = proj[x] >= threshold ? 1 : 0
|
|
}
|
|
if (rotation === 180) {
|
|
ink.reverse()
|
|
}
|
|
|
|
const tokens = cell.text.split(/\s+/).filter(Boolean)
|
|
if (tokens.length === 0) continue
|
|
|
|
const tokenWidthsPx = tokens.map(t =>
|
|
Math.max(4, Math.round(ctx.measureText(t).width * measureScale))
|
|
)
|
|
|
|
const wordPos: WordPosition[] = []
|
|
let cursor = 0
|
|
|
|
for (let ti = 0; ti < tokens.length; ti++) {
|
|
const tokenW = tokenWidthsPx[ti]
|
|
const coverageNeeded = Math.max(1, Math.round(tokenW * 0.15))
|
|
let bestX = cursor
|
|
|
|
const searchLimit = Math.max(cursor, cw - tokenW)
|
|
|
|
for (let x = cursor; x <= searchLimit; x++) {
|
|
let inkCount = 0
|
|
const spanEnd = Math.min(x + tokenW, cw)
|
|
for (let dx = 0; dx < spanEnd - x; dx++) {
|
|
inkCount += ink[x + dx]
|
|
}
|
|
if (inkCount >= coverageNeeded) {
|
|
bestX = x
|
|
break
|
|
}
|
|
if (x > cursor + cw * 0.3 && ti > 0) {
|
|
bestX = cursor
|
|
break
|
|
}
|
|
}
|
|
|
|
if (bestX + tokenW > cw) {
|
|
bestX = Math.max(0, cw - tokenW)
|
|
}
|
|
|
|
wordPos.push({
|
|
xPct: cell.bbox_pct.x + (bestX / cw) * cell.bbox_pct.w,
|
|
wPct: (tokenW / cw) * cell.bbox_pct.w,
|
|
text: tokens[ti],
|
|
fontRatio: 1.0,
|
|
})
|
|
|
|
cursor = bestX + tokenW + spaceWidthPx
|
|
}
|
|
|
|
if (wordPos.length > 0) {
|
|
positions.set(cell.cell_id, wordPos)
|
|
}
|
|
}
|
|
|
|
setResult(positions)
|
|
}
|
|
img.src = imageUrl
|
|
}, [active, cells, imageUrl, rotation])
|
|
|
|
return result
|
|
}
|