Add oversized-stub filter for large page numbers/marks in grid rows
Rows with ≤2 words, total text ≤3 chars, and word height >1.8x median are removed as non-content elements (e.g. red page number "( 9"). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -976,6 +976,9 @@ async def build_grid(session_id: str):
|
||||
# 4b. Remove junk rows: rows where ALL cells contain only short,
|
||||
# low-confidence text (OCR noise, stray marks). Real vocabulary rows
|
||||
# have at least one word with conf >= 50 or meaningful text length.
|
||||
# Also remove "oversized stub" rows: rows with ≤2 very short words
|
||||
# whose word-boxes are significantly taller than the median (e.g.
|
||||
# large red page numbers like "( 9" that are not real text content).
|
||||
_JUNK_CONF_THRESHOLD = 50
|
||||
_JUNK_MAX_TEXT_LEN = 3
|
||||
for z in zones_data:
|
||||
@@ -983,25 +986,49 @@ async def build_grid(session_id: str):
|
||||
rows = z.get("rows", [])
|
||||
if not cells or not rows:
|
||||
continue
|
||||
|
||||
# Compute median word height across the zone for oversized detection
|
||||
all_wb_heights = [
|
||||
wb["height"]
|
||||
for cell in cells
|
||||
for wb in cell.get("word_boxes") or []
|
||||
if wb.get("height", 0) > 0
|
||||
]
|
||||
median_wb_h = sorted(all_wb_heights)[len(all_wb_heights) // 2] if all_wb_heights else 28
|
||||
|
||||
junk_row_indices = set()
|
||||
for row in rows:
|
||||
ri = row["index"]
|
||||
row_cells = [c for c in cells if c.get("row_index") == ri]
|
||||
if not row_cells:
|
||||
continue
|
||||
# Check if ALL word_boxes in ALL cells of this row are junk
|
||||
|
||||
row_wbs = [
|
||||
wb for cell in row_cells
|
||||
for wb in cell.get("word_boxes") or []
|
||||
]
|
||||
|
||||
# Rule 1: ALL word_boxes are low-conf AND short text
|
||||
all_junk = True
|
||||
for cell in row_cells:
|
||||
for wb in cell.get("word_boxes") or []:
|
||||
for wb in row_wbs:
|
||||
text = (wb.get("text") or "").strip()
|
||||
conf = wb.get("conf", 0)
|
||||
if conf >= _JUNK_CONF_THRESHOLD or len(text) > _JUNK_MAX_TEXT_LEN:
|
||||
all_junk = False
|
||||
break
|
||||
if not all_junk:
|
||||
break
|
||||
if all_junk:
|
||||
if all_junk and row_wbs:
|
||||
junk_row_indices.add(ri)
|
||||
continue
|
||||
|
||||
# Rule 2: oversized stub — ≤2 words, all short text (≤2 chars),
|
||||
# and word height > 1.8× median (page numbers, stray marks)
|
||||
if len(row_wbs) <= 2:
|
||||
total_text = "".join((wb.get("text") or "").strip() for wb in row_wbs)
|
||||
max_h = max((wb.get("height", 0) for wb in row_wbs), default=0)
|
||||
if len(total_text) <= 3 and max_h > median_wb_h * 1.8:
|
||||
junk_row_indices.add(ri)
|
||||
continue
|
||||
|
||||
if junk_row_indices:
|
||||
z["cells"] = [c for c in cells if c.get("row_index") not in junk_row_indices]
|
||||
z["rows"] = [r for r in rows if r["index"] not in junk_row_indices]
|
||||
|
||||
Reference in New Issue
Block a user