feat: integrate graphic element detection into structure step
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 28s
CI / test-go-edu-search (push) Successful in 28s
CI / test-python-klausur (push) Failing after 1m58s
CI / test-python-agent-core (push) Successful in 18s
CI / test-nodejs-website (push) Successful in 19s

Add cv_graphic_detect.py for detecting non-text visual elements (arrows,
circles, lines, exclamation marks, icons, illustrations). Draw detected
graphics on structure overlay image and display them in the frontend
StepStructureDetection component with shape counts and individual listings.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-16 13:21:55 +01:00
parent 1d34785e2b
commit 6b9b280ba3
4 changed files with 447 additions and 2 deletions

View File

@@ -213,12 +213,25 @@ export interface RowGroundTruth {
notes?: string
}
export interface StructureGraphic {
x: number
y: number
w: number
h: number
area: number
shape: string // arrow, circle, line, exclamation, dot, icon, illustration
color_name: string
color_hex: string
confidence: number
}
export interface StructureResult {
image_width: number
image_height: number
content_bounds: { x: number; y: number; w: number; h: number }
boxes: StructureBox[]
zones: StructureZone[]
graphics: StructureGraphic[]
color_pixel_counts: Record<string, number>
has_words: boolean
word_count: number

View File

@@ -155,6 +155,11 @@ export function StepStructureDetection({ sessionId, onNext }: StepStructureDetec
<span className="inline-flex items-center gap-1.5 px-3 py-1 rounded-full bg-amber-50 dark:bg-amber-900/20 text-amber-700 dark:text-amber-400 text-xs font-medium">
{result.boxes.length} Box(en)
</span>
{result.graphics && result.graphics.length > 0 && (
<span className="inline-flex items-center gap-1.5 px-3 py-1 rounded-full bg-purple-50 dark:bg-purple-900/20 text-purple-700 dark:text-purple-400 text-xs font-medium">
{result.graphics.length} Grafik(en)
</span>
)}
{result.has_words && (
<span className="inline-flex items-center gap-1.5 px-3 py-1 rounded-full bg-blue-50 dark:bg-blue-900/20 text-blue-700 dark:text-blue-400 text-xs font-medium">
{result.word_count} Woerter
@@ -223,6 +228,60 @@ export function StepStructureDetection({ sessionId, onNext }: StepStructureDetec
</div>
</div>
{/* Graphics / visual elements */}
{result.graphics && result.graphics.length > 0 && (
<div>
<h4 className="text-xs font-medium text-gray-500 dark:text-gray-400 mb-2">
Graphische Elemente ({result.graphics.length})
</h4>
{/* Summary by shape */}
{(() => {
const shapeCounts: Record<string, number> = {}
for (const g of result.graphics) {
shapeCounts[g.shape] = (shapeCounts[g.shape] || 0) + 1
}
return (
<div className="flex flex-wrap gap-2 mb-2">
{Object.entries(shapeCounts)
.sort(([, a], [, b]) => b - a)
.map(([shape, count]) => (
<span
key={shape}
className="inline-flex items-center gap-1 px-2 py-1 rounded text-[11px] bg-purple-50 dark:bg-purple-900/20 text-purple-700 dark:text-purple-300 border border-purple-200 dark:border-purple-800"
>
{shape === 'arrow' ? '→' : shape === 'circle' ? '●' : shape === 'line' ? '─' : shape === 'exclamation' ? '❗' : shape === 'dot' ? '•' : shape === 'illustration' ? '🖼' : '◆'}
{' '}{shape} <span className="font-semibold">×{count}</span>
</span>
))}
</div>
)
})()}
{/* Individual graphics list */}
<div className="space-y-1.5 max-h-40 overflow-y-auto">
{result.graphics.map((g, i) => (
<div key={i} className="flex items-center gap-3 text-xs">
<span
className="w-3 h-3 rounded-full flex-shrink-0 border border-gray-300 dark:border-gray-600"
style={{ backgroundColor: g.color_hex || '#6b7280' }}
/>
<span className="text-gray-600 dark:text-gray-400 font-medium min-w-[60px]">
{g.shape}
</span>
<span className="font-mono text-gray-500">
{g.w}x{g.h}px @ ({g.x}, {g.y})
</span>
<span className="text-gray-400">
{g.color_name}
</span>
<span className="text-gray-400">
{Math.round(g.confidence * 100)}%
</span>
</div>
))}
</div>
</div>
)}
{/* Color regions */}
{Object.keys(result.color_pixel_counts).length > 0 && (
<div>