Fix prefix matching: use alpha-only chars, min 4-char prefix

Prevents false positives where punctuation (apostrophes) in merged
tokens caused wrong dictionary matches (e.g. "'se" from "'sekandarr"
matching as a word, breaking IPA continuation row fix).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-19 10:40:37 +01:00
parent 19cbbf310a
commit 21d37b5da1

View File

@@ -1063,16 +1063,18 @@ def _insert_missing_ipa(text: str, pronunciation: str = 'british') -> str:
# Fallback: prefix matching for merged tokens where OCR joined
# headword with garbled IPA (e.g. "schoolbagsku:lbæg",
# "Scotland'skotland"). Find longest dictionary prefix.
if not ipa and len(clean) > 4:
for end in range(len(clean) - 1, 2, -1):
prefix = clean[:end]
test_ipa = _lookup_ipa(prefix, pronunciation)
if test_ipa:
ipa = test_ipa
# Replace token with just the headword prefix
w = prefix
words[i] = prefix
break
# Use only alpha chars to avoid false matches on punctuation.
if not ipa:
alpha = re.sub(r'[^a-zA-Z]', '', clean)
if len(alpha) > 5: # need at least 6 chars for meaningful split
for end in range(len(alpha), 3, -1): # min prefix 4 chars
prefix = alpha[:end]
test_ipa = _lookup_ipa(prefix, pronunciation)
if test_ipa:
ipa = test_ipa
w = prefix
words[i] = prefix
break
if ipa:
words[i] = f"{w} [{ipa}]"
# Strip garbled OCR phonetics after the IPA bracket.