""" SmartSpellChecker Core — init, data types, language detection, word correction. Extracted from smart_spell.py for modularity. Lizenz: Apache 2.0 (kommerziell nutzbar) """ import logging import re from dataclasses import dataclass, field from typing import Dict, List, Literal, Optional, Set, Tuple logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Init # --------------------------------------------------------------------------- try: from spellchecker import SpellChecker as _SpellChecker _en_spell = _SpellChecker(language='en', distance=1) _de_spell = _SpellChecker(language='de', distance=1) _AVAILABLE = True except ImportError: _AVAILABLE = False logger.warning("pyspellchecker not installed — SmartSpellChecker disabled") Lang = Literal["en", "de", "both", "unknown"] # --------------------------------------------------------------------------- # Bigram context for a/I disambiguation # --------------------------------------------------------------------------- # Words that commonly follow "I" (subject pronoun -> verb/modal) _I_FOLLOWERS: frozenset = frozenset({ "am", "was", "have", "had", "do", "did", "will", "would", "can", "could", "should", "shall", "may", "might", "must", "think", "know", "see", "want", "need", "like", "love", "hate", "go", "went", "come", "came", "say", "said", "get", "got", "make", "made", "take", "took", "give", "gave", "tell", "told", "feel", "felt", "find", "found", "believe", "hope", "wish", "remember", "forget", "understand", "mean", "meant", "don't", "didn't", "can't", "won't", "couldn't", "wouldn't", "shouldn't", "haven't", "hadn't", "isn't", "wasn't", "really", "just", "also", "always", "never", "often", "sometimes", }) # Words that commonly follow "a" (article -> noun/adjective) _A_FOLLOWERS: frozenset = frozenset({ "lot", "few", "little", "bit", "good", "bad", "great", "new", "old", "long", "short", "big", "small", "large", "huge", "tiny", "nice", "beautiful", "wonderful", "terrible", "horrible", "man", "woman", "boy", "girl", "child", "dog", "cat", "bird", "book", "car", "house", "room", "school", "teacher", "student", "day", "week", "month", "year", "time", "place", "way", "friend", "family", "person", "problem", "question", "story", "very", "really", "quite", "rather", "pretty", "single", }) # Digit->letter substitutions (OCR confusion) _DIGIT_SUBS: Dict[str, List[str]] = { '0': ['o', 'O'], '1': ['l', 'I'], '5': ['s', 'S'], '6': ['g', 'G'], '8': ['b', 'B'], '|': ['I', 'l'], '/': ['l'], # italic 'l' misread as slash (e.g. "p/" -> "pl") } _SUSPICIOUS_CHARS = frozenset(_DIGIT_SUBS.keys()) # Umlaut confusion: OCR drops dots (u->u, a->a, o->o) _UMLAUT_MAP = { 'a': '\u00e4', 'o': '\u00f6', 'u': '\u00fc', 'i': '\u00fc', 'A': '\u00c4', 'O': '\u00d6', 'U': '\u00dc', 'I': '\u00dc', } # Tokenizer -- includes | and / so OCR artifacts like "p/" are treated as words _TOKEN_RE = re.compile(r"([A-Za-z\u00c4\u00d6\u00dc\u00e4\u00f6\u00fc\u00df'|/]+)([^A-Za-z\u00c4\u00d6\u00dc\u00e4\u00f6\u00fc\u00df'|/]*)") # --------------------------------------------------------------------------- # Data types # --------------------------------------------------------------------------- @dataclass class CorrectionResult: original: str corrected: str lang_detected: Lang changed: bool changes: List[str] = field(default_factory=list) # --------------------------------------------------------------------------- # Core class — language detection and word-level correction # --------------------------------------------------------------------------- class _SmartSpellCoreBase: """Base class with language detection and single-word correction. Not intended for direct use — SmartSpellChecker inherits from this. """ def __init__(self): if not _AVAILABLE: raise RuntimeError("pyspellchecker not installed") self.en = _en_spell self.de = _de_spell # --- Language detection --- def detect_word_lang(self, word: str) -> Lang: """Detect language of a single word using dual-dict heuristic.""" w = word.lower().strip(".,;:!?\"'()") if not w: return "unknown" in_en = bool(self.en.known([w])) in_de = bool(self.de.known([w])) if in_en and in_de: return "both" if in_en: return "en" if in_de: return "de" return "unknown" def detect_text_lang(self, text: str) -> Lang: """Detect dominant language of a text string (sentence/phrase).""" words = re.findall(r"[A-Za-z\u00c4\u00d6\u00dc\u00e4\u00f6\u00fc\u00df]+", text) if not words: return "unknown" en_count = 0 de_count = 0 for w in words: lang = self.detect_word_lang(w) if lang == "en": en_count += 1 elif lang == "de": de_count += 1 # "both" doesn't count for either if en_count > de_count: return "en" if de_count > en_count: return "de" if en_count == de_count and en_count > 0: return "both" return "unknown" # --- Single-word correction --- def _known(self, word: str) -> bool: """True if word is known in EN or DE dictionary, or is a known abbreviation.""" w = word.lower() if bool(self.en.known([w])) or bool(self.de.known([w])): return True # Also accept known abbreviations (sth, sb, adj, etc.) try: from cv_ocr_engines import _KNOWN_ABBREVIATIONS if w in _KNOWN_ABBREVIATIONS: return True except ImportError: pass return False def _word_freq(self, word: str) -> float: """Get word frequency (max of EN and DE).""" w = word.lower() return max(self.en.word_usage_frequency(w), self.de.word_usage_frequency(w)) def _known_in(self, word: str, lang: str) -> bool: """True if word is known in a specific language dictionary.""" w = word.lower() spell = self.en if lang == "en" else self.de return bool(spell.known([w])) def correct_word(self, word: str, lang: str = "en", prev_word: str = "", next_word: str = "") -> Optional[str]: """Correct a single word for the given language. Returns None if no correction needed, or the corrected string. """ if not word or not word.strip(): return None # Skip numbers, abbreviations with dots, very short tokens if word.isdigit() or '.' in word: return None # Skip IPA/phonetic content in brackets if '[' in word or ']' in word: return None has_suspicious = any(ch in _SUSPICIOUS_CHARS for ch in word) # 1. Already known -> no fix if self._known(word): # But check a/I disambiguation for single-char words if word.lower() in ('l', '|') and next_word: return self._disambiguate_a_I(word, next_word) return None # 2. Digit/pipe substitution if has_suspicious: if word == '|': return 'I' # Try single-char substitutions for i, ch in enumerate(word): if ch not in _DIGIT_SUBS: continue for replacement in _DIGIT_SUBS[ch]: candidate = word[:i] + replacement + word[i + 1:] if self._known(candidate): return candidate # Try multi-char substitution (e.g., "sch00l" -> "school") multi = self._try_multi_digit_sub(word) if multi: return multi # 3. Umlaut correction (German) if lang == "de" and len(word) >= 3 and word.isalpha(): umlaut_fix = self._try_umlaut_fix(word) if umlaut_fix: return umlaut_fix # 4. General spell correction if not has_suspicious and len(word) >= 3 and word.isalpha(): # Safety: don't correct if the word is valid in the OTHER language other_lang = "de" if lang == "en" else "en" if self._known_in(word, other_lang): return None if other_lang == "de" and self._try_umlaut_fix(word): return None # has a valid DE umlaut variant -> don't touch spell = self.en if lang == "en" else self.de correction = spell.correction(word.lower()) if correction and correction != word.lower(): if word[0].isupper(): correction = correction[0].upper() + correction[1:] if self._known(correction): return correction return None # --- Multi-digit substitution --- def _try_multi_digit_sub(self, word: str) -> Optional[str]: """Try replacing multiple digits simultaneously using BFS.""" positions = [(i, ch) for i, ch in enumerate(word) if ch in _DIGIT_SUBS] if not positions or len(positions) > 4: return None # BFS over substitution combinations queue = [list(word)] for pos, ch in positions: next_queue = [] for current in queue: # Keep original next_queue.append(current[:]) # Try each substitution for repl in _DIGIT_SUBS[ch]: variant = current[:] variant[pos] = repl next_queue.append(variant) queue = next_queue # Check which combinations produce known words for combo in queue: candidate = "".join(combo) if candidate != word and self._known(candidate): return candidate return None # --- Umlaut fix --- def _try_umlaut_fix(self, word: str) -> Optional[str]: """Try single-char umlaut substitutions for German words.""" for i, ch in enumerate(word): if ch in _UMLAUT_MAP: candidate = word[:i] + _UMLAUT_MAP[ch] + word[i + 1:] if self._known(candidate): return candidate return None # --- a/I disambiguation --- def _disambiguate_a_I(self, token: str, next_word: str) -> Optional[str]: """Disambiguate 'a' vs 'I' (and OCR variants like 'l', '|').""" nw = next_word.lower().strip(".,;:!?") if nw in _I_FOLLOWERS: return "I" if nw in _A_FOLLOWERS: return "a" return None # uncertain, don't change