feat(ai-sdk): vocab->tag proposer (P2 slice 5, type 3)
Extends Method C: for each unknown narrative token that pattern text names, suggest
the keyword_dictionary tag = the RequiredComponentTags shared by the naming
patterns (ranked by frequency, kept only when shared by >=40% of them, top 3).
Surfaces real dictionary gaps like "zwischenkreis" -> stored_energy and
"updates" -> has_software, which close coverage without hand-editing the dict.
Two precision fixes to Method C while here:
- patternsMentioning now matches WHOLE WORDS, not substrings — substring matching
flagged fragments like "stehen" inside "entstehen" and produced nonsensical
tag suggestions.
- a token is only proposed with a tag if one is shared by >=40% of its naming
patterns, so diffuse common verbs (spread across categories) drop out.
Wired into iace-audit propose -> audit-reports/vocab.{md,json}. Residual
common-verb noise is left to the human/LLM filter rather than a hand-grown
stopword list. Type 4 (coverage blind spots) + P3 (pin accepted proposals into a
GT case) remain for slice 6.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -66,14 +66,19 @@ func runVocabulary(form map[string]any) VocabularyReport {
|
||||
|
||||
// For each unknown token check if any pattern names it
|
||||
patterns := iace.AllPatterns()
|
||||
byID := make(map[string]iace.HazardPattern, len(patterns))
|
||||
for _, p := range patterns {
|
||||
byID[p.ID] = p
|
||||
}
|
||||
for _, tok := range report.UnknownTokens {
|
||||
hits := patternsMentioning(tok, patterns)
|
||||
if len(hits) == 0 {
|
||||
continue
|
||||
}
|
||||
report.SuggestedDictionaryEntries = append(report.SuggestedDictionaryEntries, DictionarySuggestion{
|
||||
Token: tok,
|
||||
PatternIDs: hits,
|
||||
Token: tok,
|
||||
PatternIDs: hits,
|
||||
SuggestedTags: suggestTagsFor(hits, byID),
|
||||
})
|
||||
}
|
||||
sort.Slice(report.SuggestedDictionaryEntries, func(i, j int) bool {
|
||||
@@ -129,18 +134,24 @@ func dictTokenHit(tok string, dict map[string]bool) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// patternsMentioning returns up to 8 pattern IDs whose scenario/trigger/
|
||||
// harm/zone text contains the token (case-insensitive substring).
|
||||
// patternsMentioning returns up to 8 pattern IDs whose scenario/trigger/harm/
|
||||
// zone text names the token as a WHOLE WORD. Whole-word (not substring) matching
|
||||
// is essential: a substring match flags common fragments like "stehen" inside
|
||||
// "entstehen", producing spurious hits and nonsensical tag suggestions.
|
||||
func patternsMentioning(tok string, patterns []iace.HazardPattern) []string {
|
||||
tokLower := strings.ToLower(tok)
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, p := range patterns {
|
||||
hay := strings.ToLower(p.ScenarioDE + " " + p.TriggerDE + " " + p.HarmDE + " " + p.ZoneDE + " " + p.NameDE)
|
||||
if !strings.Contains(hay, tokLower) {
|
||||
continue
|
||||
matched := false
|
||||
for _, w := range tokenRE.FindAllString(hay, -1) {
|
||||
if w == tokLower {
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if seen[p.ID] {
|
||||
if !matched || seen[p.ID] {
|
||||
continue
|
||||
}
|
||||
seen[p.ID] = true
|
||||
@@ -151,3 +162,57 @@ func patternsMentioning(tok string, patterns []iace.HazardPattern) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// suggestTagsFor returns the RequiredComponentTags shared across the naming
|
||||
// patterns, ranked by how many of them require each tag (ties broken by name),
|
||||
// top 3. These are the candidate tags a dictionary entry for the token should
|
||||
// emit so a narrative mentioning the token can trigger those patterns.
|
||||
func suggestTagsFor(ids []string, byID map[string]iace.HazardPattern) []string {
|
||||
freq := map[string]int{}
|
||||
total := 0
|
||||
for _, id := range ids {
|
||||
p, ok := byID[id]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
total++
|
||||
seen := map[string]bool{}
|
||||
for _, tag := range p.RequiredComponentTags {
|
||||
if seen[tag] {
|
||||
continue
|
||||
}
|
||||
seen[tag] = true
|
||||
freq[tag]++
|
||||
}
|
||||
}
|
||||
if total == 0 {
|
||||
return nil
|
||||
}
|
||||
type tf struct {
|
||||
tag string
|
||||
n int
|
||||
}
|
||||
ranked := make([]tf, 0, len(freq))
|
||||
for t, n := range freq {
|
||||
ranked = append(ranked, tf{t, n})
|
||||
}
|
||||
sort.Slice(ranked, func(i, j int) bool {
|
||||
if ranked[i].n != ranked[j].n {
|
||||
return ranked[i].n > ranked[j].n
|
||||
}
|
||||
return ranked[i].tag < ranked[j].tag
|
||||
})
|
||||
// Only suggest a tag shared by >= 40% of the naming patterns. Diffuse tokens
|
||||
// (common verbs spread across categories) get no dominant tag and are dropped.
|
||||
var out []string
|
||||
for _, x := range ranked {
|
||||
if float64(x.n)/float64(total) < 0.4 {
|
||||
break
|
||||
}
|
||||
out = append(out, x.tag)
|
||||
if len(out) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user