bd4b956e3c
klausur-service (11 files): - cv_gutter_repair, ocr_pipeline_regression, upload_api - ocr_pipeline_sessions, smart_spell, nru_worksheet_generator - ocr_pipeline_overlays, mail/aggregator, zeugnis_api - cv_syllable_detect, self_rag backend-lehrer (17 files): - classroom_engine/suggestions, generators/quiz_generator - worksheets_api, llm_gateway/comparison, state_engine_api - classroom/models (→ 4 submodules), services/file_processor - alerts_agent/api/wizard+digests+routes, content_generators/pdf - classroom/routes/sessions, llm_gateway/inference - classroom_engine/analytics, auth/keycloak_auth - alerts_agent/processing/rule_engine, ai_processor/print_versions agent-core (5 files): - brain/memory_store, brain/knowledge_graph, brain/context_manager - orchestrator/supervisor, sessions/session_manager admin-lehrer (5 components): - GridOverlay, StepGridReview, DevOpsPipelineSidebar - DataFlowDiagram, sbom/wizard/page website (2 files): - DependencyMap, lehrer/abitur-archiv Other: nibis_ingestion, grid_detection_service, export-doclayout-onnx Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
299 lines
11 KiB
Python
299 lines
11 KiB
Python
"""
|
|
SmartSpellChecker Core — init, data types, language detection, word correction.
|
|
|
|
Extracted from smart_spell.py for modularity.
|
|
|
|
Lizenz: Apache 2.0 (kommerziell nutzbar)
|
|
"""
|
|
|
|
import logging
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from typing import Dict, List, Literal, Optional, Set, Tuple
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Init
|
|
# ---------------------------------------------------------------------------
|
|
|
|
try:
|
|
from spellchecker import SpellChecker as _SpellChecker
|
|
_en_spell = _SpellChecker(language='en', distance=1)
|
|
_de_spell = _SpellChecker(language='de', distance=1)
|
|
_AVAILABLE = True
|
|
except ImportError:
|
|
_AVAILABLE = False
|
|
logger.warning("pyspellchecker not installed — SmartSpellChecker disabled")
|
|
|
|
Lang = Literal["en", "de", "both", "unknown"]
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bigram context for a/I disambiguation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Words that commonly follow "I" (subject pronoun -> verb/modal)
|
|
_I_FOLLOWERS: frozenset = frozenset({
|
|
"am", "was", "have", "had", "do", "did", "will", "would", "can",
|
|
"could", "should", "shall", "may", "might", "must",
|
|
"think", "know", "see", "want", "need", "like", "love", "hate",
|
|
"go", "went", "come", "came", "say", "said", "get", "got",
|
|
"make", "made", "take", "took", "give", "gave", "tell", "told",
|
|
"feel", "felt", "find", "found", "believe", "hope", "wish",
|
|
"remember", "forget", "understand", "mean", "meant",
|
|
"don't", "didn't", "can't", "won't", "couldn't", "wouldn't",
|
|
"shouldn't", "haven't", "hadn't", "isn't", "wasn't",
|
|
"really", "just", "also", "always", "never", "often", "sometimes",
|
|
})
|
|
|
|
# Words that commonly follow "a" (article -> noun/adjective)
|
|
_A_FOLLOWERS: frozenset = frozenset({
|
|
"lot", "few", "little", "bit", "good", "bad", "great", "new", "old",
|
|
"long", "short", "big", "small", "large", "huge", "tiny",
|
|
"nice", "beautiful", "wonderful", "terrible", "horrible",
|
|
"man", "woman", "boy", "girl", "child", "dog", "cat", "bird",
|
|
"book", "car", "house", "room", "school", "teacher", "student",
|
|
"day", "week", "month", "year", "time", "place", "way",
|
|
"friend", "family", "person", "problem", "question", "story",
|
|
"very", "really", "quite", "rather", "pretty", "single",
|
|
})
|
|
|
|
# Digit->letter substitutions (OCR confusion)
|
|
_DIGIT_SUBS: Dict[str, List[str]] = {
|
|
'0': ['o', 'O'],
|
|
'1': ['l', 'I'],
|
|
'5': ['s', 'S'],
|
|
'6': ['g', 'G'],
|
|
'8': ['b', 'B'],
|
|
'|': ['I', 'l'],
|
|
'/': ['l'], # italic 'l' misread as slash (e.g. "p/" -> "pl")
|
|
}
|
|
_SUSPICIOUS_CHARS = frozenset(_DIGIT_SUBS.keys())
|
|
|
|
# Umlaut confusion: OCR drops dots (u->u, a->a, o->o)
|
|
_UMLAUT_MAP = {
|
|
'a': '\u00e4', 'o': '\u00f6', 'u': '\u00fc', 'i': '\u00fc',
|
|
'A': '\u00c4', 'O': '\u00d6', 'U': '\u00dc', 'I': '\u00dc',
|
|
}
|
|
|
|
# Tokenizer -- includes | and / so OCR artifacts like "p/" are treated as words
|
|
_TOKEN_RE = re.compile(r"([A-Za-z\u00c4\u00d6\u00dc\u00e4\u00f6\u00fc\u00df'|/]+)([^A-Za-z\u00c4\u00d6\u00dc\u00e4\u00f6\u00fc\u00df'|/]*)")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Data types
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class CorrectionResult:
|
|
original: str
|
|
corrected: str
|
|
lang_detected: Lang
|
|
changed: bool
|
|
changes: List[str] = field(default_factory=list)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Core class — language detection and word-level correction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _SmartSpellCoreBase:
|
|
"""Base class with language detection and single-word correction.
|
|
|
|
Not intended for direct use — SmartSpellChecker inherits from this.
|
|
"""
|
|
|
|
def __init__(self):
|
|
if not _AVAILABLE:
|
|
raise RuntimeError("pyspellchecker not installed")
|
|
self.en = _en_spell
|
|
self.de = _de_spell
|
|
|
|
# --- Language detection ---
|
|
|
|
def detect_word_lang(self, word: str) -> Lang:
|
|
"""Detect language of a single word using dual-dict heuristic."""
|
|
w = word.lower().strip(".,;:!?\"'()")
|
|
if not w:
|
|
return "unknown"
|
|
in_en = bool(self.en.known([w]))
|
|
in_de = bool(self.de.known([w]))
|
|
if in_en and in_de:
|
|
return "both"
|
|
if in_en:
|
|
return "en"
|
|
if in_de:
|
|
return "de"
|
|
return "unknown"
|
|
|
|
def detect_text_lang(self, text: str) -> Lang:
|
|
"""Detect dominant language of a text string (sentence/phrase)."""
|
|
words = re.findall(r"[A-Za-z\u00c4\u00d6\u00dc\u00e4\u00f6\u00fc\u00df]+", text)
|
|
if not words:
|
|
return "unknown"
|
|
|
|
en_count = 0
|
|
de_count = 0
|
|
for w in words:
|
|
lang = self.detect_word_lang(w)
|
|
if lang == "en":
|
|
en_count += 1
|
|
elif lang == "de":
|
|
de_count += 1
|
|
# "both" doesn't count for either
|
|
|
|
if en_count > de_count:
|
|
return "en"
|
|
if de_count > en_count:
|
|
return "de"
|
|
if en_count == de_count and en_count > 0:
|
|
return "both"
|
|
return "unknown"
|
|
|
|
# --- Single-word correction ---
|
|
|
|
def _known(self, word: str) -> bool:
|
|
"""True if word is known in EN or DE dictionary, or is a known abbreviation."""
|
|
w = word.lower()
|
|
if bool(self.en.known([w])) or bool(self.de.known([w])):
|
|
return True
|
|
# Also accept known abbreviations (sth, sb, adj, etc.)
|
|
try:
|
|
from cv_ocr_engines import _KNOWN_ABBREVIATIONS
|
|
if w in _KNOWN_ABBREVIATIONS:
|
|
return True
|
|
except ImportError:
|
|
pass
|
|
return False
|
|
|
|
def _word_freq(self, word: str) -> float:
|
|
"""Get word frequency (max of EN and DE)."""
|
|
w = word.lower()
|
|
return max(self.en.word_usage_frequency(w), self.de.word_usage_frequency(w))
|
|
|
|
def _known_in(self, word: str, lang: str) -> bool:
|
|
"""True if word is known in a specific language dictionary."""
|
|
w = word.lower()
|
|
spell = self.en if lang == "en" else self.de
|
|
return bool(spell.known([w]))
|
|
|
|
def correct_word(self, word: str, lang: str = "en",
|
|
prev_word: str = "", next_word: str = "") -> Optional[str]:
|
|
"""Correct a single word for the given language.
|
|
|
|
Returns None if no correction needed, or the corrected string.
|
|
"""
|
|
if not word or not word.strip():
|
|
return None
|
|
|
|
# Skip numbers, abbreviations with dots, very short tokens
|
|
if word.isdigit() or '.' in word:
|
|
return None
|
|
|
|
# Skip IPA/phonetic content in brackets
|
|
if '[' in word or ']' in word:
|
|
return None
|
|
|
|
has_suspicious = any(ch in _SUSPICIOUS_CHARS for ch in word)
|
|
|
|
# 1. Already known -> no fix
|
|
if self._known(word):
|
|
# But check a/I disambiguation for single-char words
|
|
if word.lower() in ('l', '|') and next_word:
|
|
return self._disambiguate_a_I(word, next_word)
|
|
return None
|
|
|
|
# 2. Digit/pipe substitution
|
|
if has_suspicious:
|
|
if word == '|':
|
|
return 'I'
|
|
# Try single-char substitutions
|
|
for i, ch in enumerate(word):
|
|
if ch not in _DIGIT_SUBS:
|
|
continue
|
|
for replacement in _DIGIT_SUBS[ch]:
|
|
candidate = word[:i] + replacement + word[i + 1:]
|
|
if self._known(candidate):
|
|
return candidate
|
|
# Try multi-char substitution (e.g., "sch00l" -> "school")
|
|
multi = self._try_multi_digit_sub(word)
|
|
if multi:
|
|
return multi
|
|
|
|
# 3. Umlaut correction (German)
|
|
if lang == "de" and len(word) >= 3 and word.isalpha():
|
|
umlaut_fix = self._try_umlaut_fix(word)
|
|
if umlaut_fix:
|
|
return umlaut_fix
|
|
|
|
# 4. General spell correction
|
|
if not has_suspicious and len(word) >= 3 and word.isalpha():
|
|
# Safety: don't correct if the word is valid in the OTHER language
|
|
other_lang = "de" if lang == "en" else "en"
|
|
if self._known_in(word, other_lang):
|
|
return None
|
|
if other_lang == "de" and self._try_umlaut_fix(word):
|
|
return None # has a valid DE umlaut variant -> don't touch
|
|
|
|
spell = self.en if lang == "en" else self.de
|
|
correction = spell.correction(word.lower())
|
|
if correction and correction != word.lower():
|
|
if word[0].isupper():
|
|
correction = correction[0].upper() + correction[1:]
|
|
if self._known(correction):
|
|
return correction
|
|
|
|
return None
|
|
|
|
# --- Multi-digit substitution ---
|
|
|
|
def _try_multi_digit_sub(self, word: str) -> Optional[str]:
|
|
"""Try replacing multiple digits simultaneously using BFS."""
|
|
positions = [(i, ch) for i, ch in enumerate(word) if ch in _DIGIT_SUBS]
|
|
if not positions or len(positions) > 4:
|
|
return None
|
|
|
|
# BFS over substitution combinations
|
|
queue = [list(word)]
|
|
for pos, ch in positions:
|
|
next_queue = []
|
|
for current in queue:
|
|
# Keep original
|
|
next_queue.append(current[:])
|
|
# Try each substitution
|
|
for repl in _DIGIT_SUBS[ch]:
|
|
variant = current[:]
|
|
variant[pos] = repl
|
|
next_queue.append(variant)
|
|
queue = next_queue
|
|
|
|
# Check which combinations produce known words
|
|
for combo in queue:
|
|
candidate = "".join(combo)
|
|
if candidate != word and self._known(candidate):
|
|
return candidate
|
|
|
|
return None
|
|
|
|
# --- Umlaut fix ---
|
|
|
|
def _try_umlaut_fix(self, word: str) -> Optional[str]:
|
|
"""Try single-char umlaut substitutions for German words."""
|
|
for i, ch in enumerate(word):
|
|
if ch in _UMLAUT_MAP:
|
|
candidate = word[:i] + _UMLAUT_MAP[ch] + word[i + 1:]
|
|
if self._known(candidate):
|
|
return candidate
|
|
return None
|
|
|
|
# --- a/I disambiguation ---
|
|
|
|
def _disambiguate_a_I(self, token: str, next_word: str) -> Optional[str]:
|
|
"""Disambiguate 'a' vs 'I' (and OCR variants like 'l', '|')."""
|
|
nw = next_word.lower().strip(".,;:!?")
|
|
if nw in _I_FOLLOWERS:
|
|
return "I"
|
|
if nw in _A_FOLLOWERS:
|
|
return "a"
|
|
return None # uncertain, don't change
|