fix(rag): strip HTML tags before chunking + D5 re-ingestion scripts

HTML files from gesetze-im-internet.de were decoded as raw UTF-8, keeping <div>/<p> tags intact. The legal chunker regex requires § at line start, which never matched inside HTML tags → 0% section metadata for HTML docs. Fix: detect HTML content and strip tags before sending to embedding service. Block elements become newlines, entities are decoded. § signs now appear at line starts → section detection works. Also adds D5 re-ingestion scripts (reingest_d5.py + config) for batch re-processing of all documents in Qdrant collections. 27 rag-service tests passing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-05-02 08:18:25 +02:00
parent 93099b2770
commit ddad58f607
5 changed files with 698 additions and 0 deletions
@@ -7,6 +7,7 @@ from pydantic import BaseModel

 from api.auth import optional_jwt_auth
 from embedding_client import embedding_client
+from html_utils import looks_like_html, strip_html
 from minio_client_wrapper import minio_wrapper
 from qdrant_client_wrapper import qdrant_wrapper

@@ -111,6 +112,11 @@ async def upload_document(
    if not text or not text.strip():
        raise HTTPException(status_code=400, detail="Could not extract any text from the document")

+    # --- Strip HTML if detected ---
+    if looks_like_html(text):
+        text = strip_html(text)
+        logger.info("Stripped HTML tags from %s", filename)
+
    # --- Chunk ---
    try:
        chunk_result = await embedding_client.chunk_text(
@@ -0,0 +1,31 @@
+"""HTML detection and stripping for legal document ingestion."""
+
+import re
+from html import unescape
+
+_HTML_TAG_RE = re.compile(r'<(html|head|body|div|p|span|table)\b', re.IGNORECASE)
+
+
+def looks_like_html(text: str) -> bool:
+    """Check if text contains HTML tags."""
+    return bool(_HTML_TAG_RE.search(text[:500]))
+
+
+def strip_html(html_text: str) -> str:
+    """Convert HTML to plain text preserving legal document structure."""
+    text = html_text
+    # Remove script/style blocks
+    text = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', text, flags=re.DOTALL | re.IGNORECASE)
+    # Block elements → newline (preserves § paragraph structure)
+    text = re.sub(
+        r'</(div|p|h[1-6]|li|tr|section|article|blockquote)>',
+        '\n', text, flags=re.IGNORECASE,
+    )
+    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
+    # Strip remaining tags
+    text = re.sub(r'<[^>]+>', '', text)
+    # Decode HTML entities (&#246; → ö, &sect; → §)
+    text = unescape(text)
+    # Clean up excessive whitespace
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text.strip()
@@ -0,0 +1,122 @@
+"""Tests for HTML detection and stripping in document upload."""
+
+from html_utils import looks_like_html as _looks_like_html, strip_html as _strip_html
+
+
+class TestLooksLikeHtml:
+
+    def test_html_document(self):
+        assert _looks_like_html("<html><body><p>Text</p></body></html>")
+
+    def test_html_div(self):
+        assert _looks_like_html('<div class="jurAbsatz">§ 312</div>')
+
+    def test_html_with_doctype(self):
+        assert _looks_like_html("<!DOCTYPE html><html><head></head><body>")
+
+    def test_plain_text(self):
+        assert not _looks_like_html("§ 312 Anwendungsbereich\n\n(1) Die Vorschriften...")
+
+    def test_legal_text_with_angle_brackets(self):
+        # Legal text might use < or > but not as HTML tags
+        assert not _looks_like_html("Wert < 100 EUR und > 50 EUR ist zulaessig.")
+
+    def test_markdown(self):
+        assert not _looks_like_html("# § 312 Anwendungsbereich\n\n(1) Die Vorschriften...")
+
+
+class TestStripHtml:
+
+    def test_basic_div_tags(self):
+        html = "<div>§ 312 Anwendungsbereich</div>"
+        result = _strip_html(html)
+        assert result.startswith("§ 312 Anwendungsbereich")
+
+    def test_paragraph_tags_become_newlines(self):
+        html = "<p>Absatz 1</p><p>Absatz 2</p>"
+        result = _strip_html(html)
+        assert "Absatz 1" in result
+        assert "Absatz 2" in result
+        # Paragraphs should be on separate lines
+        lines = [ln.strip() for ln in result.split("\n") if ln.strip()]
+        assert len(lines) >= 2
+
+    def test_preserves_section_headers(self):
+        """§ signs must be at line starts after stripping."""
+        html = '<div class="jurAbsatz">§ 312 Anwendungsbereich</div>'
+        result = _strip_html(html)
+        # § should be at the start of a line
+        for line in result.split("\n"):
+            if "§ 312" in line:
+                assert line.strip().startswith("§ 312")
+                break
+        else:
+            raise AssertionError("§ 312 not found in stripped text")
+
+    def test_decodes_html_entities(self):
+        html = "Gel&#246;scht und ge&#228;ndert und &#167; 312"
+        result = _strip_html(html)
+        assert "Gelöscht" in result
+        assert "geändert" in result
+        assert "§ 312" in result
+
+    def test_decodes_named_entities(self):
+        html = "&sect; 312 &amp; &sect; 313"
+        result = _strip_html(html)
+        assert "§ 312" in result
+        assert "§ 313" in result
+
+    def test_removes_script_style(self):
+        html = '<style>body{color:red}</style><script>alert("x")</script><p>§ 1 Text</p>'
+        result = _strip_html(html)
+        assert "color" not in result
+        assert "alert" not in result
+        assert "§ 1 Text" in result
+
+    def test_br_becomes_newline(self):
+        html = "Zeile 1<br/>Zeile 2<br>Zeile 3"
+        result = _strip_html(html)
+        assert "Zeile 1" in result
+        assert "Zeile 2" in result
+
+    def test_no_excessive_whitespace(self):
+        html = "<div></div><div></div><div></div><div>Text</div>"
+        result = _strip_html(html)
+        assert "\n\n\n" not in result
+
+    def test_gesetze_im_internet_format(self):
+        """Realistic HTML from gesetze-im-internet.de."""
+        html = """<div class="jnhtml">
+<div>
+<div class="jurAbsatz">
+§ 312k Kündigung von Verbraucherverträgen im elektronischen Geschäftsverkehr
+</div>
+<div class="jurAbsatz">
+(1) Wird Verbrauchern über eine Webseite ermöglicht, einen Vertrag im elektronischen Geschäftsverkehr zu schließen, der auf die Begründung eines Dauerschuldverhältnisses gerichtet ist, das einen Unternehmer zu einer entgeltlichen Leistung verpflichtet, so treffen den Unternehmer die Pflichten nach dieser Vorschrift.
+</div>
+<div class="jurAbsatz">
+(2) Der Unternehmer hat sicherzustellen, dass der Verbraucher auf der Webseite eine Erklärung zur ordentlichen oder außerordentlichen Kündigung abgeben kann.
+</div>
+</div></div>"""
+        result = _strip_html(html)
+
+        # § 312k should be at start of a line
+        found_312k = False
+        for line in result.split("\n"):
+            stripped = line.strip()
+            if stripped.startswith("§ 312k"):
+                found_312k = True
+                break
+        assert found_312k, f"§ 312k not at line start. Text:\n{result[:500]}"
+
+        # Content should be present without tags
+        assert "Dauerschuldverhältnisses" in result
+        assert "<div>" not in result
+        assert "class=" not in result
+
+    def test_plain_text_passthrough(self):
+        """Non-HTML text should pass through unchanged."""
+        text = "§ 312 Anwendungsbereich\n\n(1) Die Vorschriften..."
+        result = _strip_html(text)
+        assert "§ 312 Anwendungsbereich" in result
+        assert "(1) Die Vorschriften" in result