feat: integrate graphic element detection into structure step
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 28s
CI / test-go-edu-search (push) Successful in 28s
CI / test-python-klausur (push) Failing after 1m58s
CI / test-python-agent-core (push) Successful in 18s
CI / test-nodejs-website (push) Successful in 19s
Some checks failed
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-school (push) Successful in 28s
CI / test-go-edu-search (push) Successful in 28s
CI / test-python-klausur (push) Failing after 1m58s
CI / test-python-agent-core (push) Successful in 18s
CI / test-nodejs-website (push) Successful in 19s
Add cv_graphic_detect.py for detecting non-text visual elements (arrows, circles, lines, exclamation marks, icons, illustrations). Draw detected graphics on structure overlay image and display them in the frontend StepStructureDetection component with shape counts and individual listings. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -213,12 +213,25 @@ export interface RowGroundTruth {
|
||||
notes?: string
|
||||
}
|
||||
|
||||
export interface StructureGraphic {
|
||||
x: number
|
||||
y: number
|
||||
w: number
|
||||
h: number
|
||||
area: number
|
||||
shape: string // arrow, circle, line, exclamation, dot, icon, illustration
|
||||
color_name: string
|
||||
color_hex: string
|
||||
confidence: number
|
||||
}
|
||||
|
||||
export interface StructureResult {
|
||||
image_width: number
|
||||
image_height: number
|
||||
content_bounds: { x: number; y: number; w: number; h: number }
|
||||
boxes: StructureBox[]
|
||||
zones: StructureZone[]
|
||||
graphics: StructureGraphic[]
|
||||
color_pixel_counts: Record<string, number>
|
||||
has_words: boolean
|
||||
word_count: number
|
||||
|
||||
@@ -155,6 +155,11 @@ export function StepStructureDetection({ sessionId, onNext }: StepStructureDetec
|
||||
<span className="inline-flex items-center gap-1.5 px-3 py-1 rounded-full bg-amber-50 dark:bg-amber-900/20 text-amber-700 dark:text-amber-400 text-xs font-medium">
|
||||
{result.boxes.length} Box(en)
|
||||
</span>
|
||||
{result.graphics && result.graphics.length > 0 && (
|
||||
<span className="inline-flex items-center gap-1.5 px-3 py-1 rounded-full bg-purple-50 dark:bg-purple-900/20 text-purple-700 dark:text-purple-400 text-xs font-medium">
|
||||
{result.graphics.length} Grafik(en)
|
||||
</span>
|
||||
)}
|
||||
{result.has_words && (
|
||||
<span className="inline-flex items-center gap-1.5 px-3 py-1 rounded-full bg-blue-50 dark:bg-blue-900/20 text-blue-700 dark:text-blue-400 text-xs font-medium">
|
||||
{result.word_count} Woerter
|
||||
@@ -223,6 +228,60 @@ export function StepStructureDetection({ sessionId, onNext }: StepStructureDetec
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Graphics / visual elements */}
|
||||
{result.graphics && result.graphics.length > 0 && (
|
||||
<div>
|
||||
<h4 className="text-xs font-medium text-gray-500 dark:text-gray-400 mb-2">
|
||||
Graphische Elemente ({result.graphics.length})
|
||||
</h4>
|
||||
{/* Summary by shape */}
|
||||
{(() => {
|
||||
const shapeCounts: Record<string, number> = {}
|
||||
for (const g of result.graphics) {
|
||||
shapeCounts[g.shape] = (shapeCounts[g.shape] || 0) + 1
|
||||
}
|
||||
return (
|
||||
<div className="flex flex-wrap gap-2 mb-2">
|
||||
{Object.entries(shapeCounts)
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.map(([shape, count]) => (
|
||||
<span
|
||||
key={shape}
|
||||
className="inline-flex items-center gap-1 px-2 py-1 rounded text-[11px] bg-purple-50 dark:bg-purple-900/20 text-purple-700 dark:text-purple-300 border border-purple-200 dark:border-purple-800"
|
||||
>
|
||||
{shape === 'arrow' ? '→' : shape === 'circle' ? '●' : shape === 'line' ? '─' : shape === 'exclamation' ? '❗' : shape === 'dot' ? '•' : shape === 'illustration' ? '🖼' : '◆'}
|
||||
{' '}{shape} <span className="font-semibold">×{count}</span>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
})()}
|
||||
{/* Individual graphics list */}
|
||||
<div className="space-y-1.5 max-h-40 overflow-y-auto">
|
||||
{result.graphics.map((g, i) => (
|
||||
<div key={i} className="flex items-center gap-3 text-xs">
|
||||
<span
|
||||
className="w-3 h-3 rounded-full flex-shrink-0 border border-gray-300 dark:border-gray-600"
|
||||
style={{ backgroundColor: g.color_hex || '#6b7280' }}
|
||||
/>
|
||||
<span className="text-gray-600 dark:text-gray-400 font-medium min-w-[60px]">
|
||||
{g.shape}
|
||||
</span>
|
||||
<span className="font-mono text-gray-500">
|
||||
{g.w}x{g.h}px @ ({g.x}, {g.y})
|
||||
</span>
|
||||
<span className="text-gray-400">
|
||||
{g.color_name}
|
||||
</span>
|
||||
<span className="text-gray-400">
|
||||
{Math.round(g.confidence * 100)}%
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Color regions */}
|
||||
{Object.keys(result.color_pixel_counts).length > 0 && (
|
||||
<div>
|
||||
|
||||
Reference in New Issue
Block a user