feat(ocr-pipeline): generic header/footer detection via projection gap analysis

Replace the trivial top_y/bottom_y threshold check with horizontal
projection gap analysis that finds large whitespace gaps separating
header/footer content from the main body. This correctly detects
headers (e.g. "VOCABULARY" banners) and footers (page numbers) even
when _find_content_bounds includes them in the content area.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

klausur-service/backend/cv_vocab_pipeline.py

@@ -921,22 +921,15 @@ def analyze_layout(layout_img: np.ndarray, ocr_img: np.ndarray) -> List[PageRegi
             width=w, height=content_h
         ))
 
-    # Add header/footer info
-    if top_y > 10:
-        regions.append(PageRegion(
-            type='header', x=0, y=0,
-            width=w, height=top_y
-        ))
-    if bottom_y < h - 10:
-        regions.append(PageRegion(
-            type='footer', x=0, y=bottom_y,
-            width=w, height=h - bottom_y
-        ))
+    # Add header/footer info (gap-based detection with fallback)
+    _add_header_footer(regions, top_y, bottom_y, w, h, inv=inv)
 
+    has_header = any(r.type == 'header' for r in regions)
+    has_footer = any(r.type == 'footer' for r in regions)
     col_count = len([r for r in regions if r.type.startswith('column')])
     logger.info(f"Layout: {col_count} columns, "
-                f"header={'yes' if top_y > 10 else 'no'}, "
-                f"footer={'yes' if bottom_y < h - 10 else 'no'}")
+                f"header={'yes' if has_header else 'no'}, "
+                f"footer={'yes' if has_footer else 'no'}")
 
     return regions
 
@@ -2076,7 +2069,8 @@ def classify_column_types(geometries: List[ColumnGeometry],
                           img_h: int,
                           bottom_y: int,
                           left_x: int = 0,
-                          right_x: int = 0) -> List[PageRegion]:
+                          right_x: int = 0,
+                          inv: Optional[np.ndarray] = None) -> List[PageRegion]:
     """Classify column types using a 3-level fallback chain.
 
     Level 1: Content-based (language + role scoring)
@@ -2159,20 +2153,20 @@ def classify_column_types(geometries: List[ColumnGeometry],
     regions = _classify_by_content(geometries, lang_scores, role_scores, content_w, content_h)
     if regions is not None:
         logger.info("ClassifyColumns: Level 1 (content-based) succeeded")
-        _add_header_footer(regions, top_y, bottom_y, img_w, img_h)
+        _add_header_footer(regions, top_y, bottom_y, img_w, img_h, inv=inv)
         return _with_margins(ignore_regions + regions)
 
     # --- Level 2: Position + language enhanced ---
     regions = _classify_by_position_enhanced(geometries, lang_scores, content_w, content_h)
     if regions is not None:
         logger.info("ClassifyColumns: Level 2 (position+language) succeeded")
-        _add_header_footer(regions, top_y, bottom_y, img_w, img_h)
+        _add_header_footer(regions, top_y, bottom_y, img_w, img_h, inv=inv)
         return _with_margins(ignore_regions + regions)
 
     # --- Level 3: Pure position fallback (old code, no regression) ---
     logger.info("ClassifyColumns: Level 3 (position fallback)")
     regions = _classify_by_position_fallback(geometries, content_w, content_h)
-    _add_header_footer(regions, top_y, bottom_y, img_w, img_h)
+    _add_header_footer(regions, top_y, bottom_y, img_w, img_h, inv=inv)
     return _with_margins(ignore_regions + regions)
 
 
@@ -2534,12 +2528,127 @@ def _classify_by_position_fallback(geometries: List[ColumnGeometry],
     return regions
 
 
+def _detect_header_footer_gaps(
+    inv: np.ndarray,
+    img_w: int,
+    img_h: int,
+) -> Tuple[Optional[int], Optional[int]]:
+    """Detect header/footer boundaries via horizontal projection gap analysis.
+
+    Scans the full-page inverted image for large horizontal gaps in the top/bottom
+    20% that separate header/footer content from the main body.
+
+    Returns:
+        (header_y, footer_y) — absolute y-coordinates.
+        header_y = bottom edge of header region (None if no header detected).
+        footer_y = top edge of footer region (None if no footer detected).
+    """
+    HEADER_FOOTER_ZONE = 0.20
+    GAP_MULTIPLIER = 2.0
+
+    # Step 1: Horizontal projection over full image width
+    h_proj = np.sum(inv, axis=1).astype(float)
+    h_proj_norm = h_proj / (img_w * 255) if img_w > 0 else h_proj
+
+    # Step 2: Smoothing
+    kernel_size = max(3, img_h // 200)
+    if kernel_size % 2 == 0:
+        kernel_size += 1
+    h_smooth = np.convolve(h_proj_norm, np.ones(kernel_size) / kernel_size, mode='same')
+
+    # Step 3: Gap threshold
+    positive = h_smooth[h_smooth > 0]
+    median_density = float(np.median(positive)) if len(positive) > 0 else 0.01
+    gap_threshold = max(median_density * 0.15, 0.003)
+
+    in_gap = h_smooth < gap_threshold
+    MIN_GAP_HEIGHT = max(3, img_h // 500)
+
+    # Step 4: Collect contiguous gaps
+    raw_gaps: List[Tuple[int, int]] = []
+    gap_start: Optional[int] = None
+    for y in range(len(in_gap)):
+        if in_gap[y]:
+            if gap_start is None:
+                gap_start = y
+        else:
+            if gap_start is not None:
+                gap_height = y - gap_start
+                if gap_height >= MIN_GAP_HEIGHT:
+                    raw_gaps.append((gap_start, y))
+                gap_start = None
+    if gap_start is not None:
+        gap_height = len(in_gap) - gap_start
+        if gap_height >= MIN_GAP_HEIGHT:
+            raw_gaps.append((gap_start, len(in_gap)))
+
+    if not raw_gaps:
+        return None, None
+
+    # Step 5: Compute median gap size and large-gap threshold
+    gap_sizes = [g[1] - g[0] for g in raw_gaps]
+    median_gap = float(np.median(gap_sizes))
+    large_gap_threshold = median_gap * GAP_MULTIPLIER
+
+    # Step 6: Find largest qualifying gap in header / footer zones
+    header_zone_limit = int(img_h * HEADER_FOOTER_ZONE)
+    footer_zone_start = int(img_h * (1.0 - HEADER_FOOTER_ZONE))
+
+    header_y: Optional[int] = None
+    footer_y: Optional[int] = None
+
+    best_header_size = 0
+    for gs, ge in raw_gaps:
+        gap_mid = (gs + ge) / 2
+        gap_size = ge - gs
+        if gap_mid < header_zone_limit and gap_size > large_gap_threshold:
+            if gap_size > best_header_size:
+                best_header_size = gap_size
+                header_y = ge  # bottom edge of gap
+
+    best_footer_size = 0
+    for gs, ge in raw_gaps:
+        gap_mid = (gs + ge) / 2
+        gap_size = ge - gs
+        if gap_mid > footer_zone_start and gap_size > large_gap_threshold:
+            if gap_size > best_footer_size:
+                best_footer_size = gap_size
+                footer_y = gs  # top edge of gap
+
+    if header_y is not None:
+        logger.info(f"HeaderFooterGaps: header boundary at y={header_y} "
+                    f"(gap={best_header_size}px, median_gap={median_gap:.0f}px)")
+    if footer_y is not None:
+        logger.info(f"HeaderFooterGaps: footer boundary at y={footer_y} "
+                    f"(gap={best_footer_size}px, median_gap={median_gap:.0f}px)")
+
+    return header_y, footer_y
+
+
 def _add_header_footer(regions: List[PageRegion], top_y: int, bottom_y: int,
-                       img_w: int, img_h: int) -> None:
-    """Add header/footer regions in-place."""
-    if top_y > 10:
+                       img_w: int, img_h: int,
+                       inv: Optional[np.ndarray] = None) -> None:
+    """Add header/footer regions in-place.
+
+    When *inv* is provided, uses gap-based detection to find header/footer
+    boundaries. Falls back to simple top_y/bottom_y check otherwise.
+    """
+    header_y: Optional[int] = None
+    footer_y: Optional[int] = None
+
+    if inv is not None:
+        header_y, footer_y = _detect_header_footer_gaps(inv, img_w, img_h)
+
+    # Gap-based header
+    if header_y is not None and header_y > 10:
+        regions.append(PageRegion(type='header', x=0, y=0, width=img_w, height=header_y))
+    elif top_y > 10:
         regions.append(PageRegion(type='header', x=0, y=0, width=img_w, height=top_y))
-    if bottom_y < img_h - 10:
+
+    # Gap-based footer
+    if footer_y is not None and footer_y < img_h - 10:
+        regions.append(PageRegion(type='footer', x=0, y=footer_y, width=img_w, height=img_h - footer_y))
+    elif bottom_y < img_h - 10:
         regions.append(PageRegion(type='footer', x=0, y=bottom_y, width=img_w, height=img_h - bottom_y))
 
 
@@ -2576,7 +2685,7 @@ def analyze_layout_by_words(ocr_img: np.ndarray, dewarped_bgr: np.ndarray) -> Li
 
     # Phase B: Content-based classification
     regions = classify_column_types(geometries, content_w, top_y, w, h, bottom_y,
-                                    left_x=left_x, right_x=right_x)
+                                    left_x=left_x, right_x=right_x, inv=_inv)
 
     col_count = len([r for r in regions if r.type.startswith('column') or r.type == 'page_ref'])
     methods = set(r.classification_method for r in regions if r.classification_method)