package iace import ( "encoding/json" "os" "path/filepath" "sort" "strings" "testing" ) // ============================================================================ // Cross-GT real-narrative benchmark harness. // // Unlike gt_kistenhub_test.go (which feeds a hand-built MatchInput), this // harness runs the FULL production pipeline: machine narrative → ParseNarrative // → MatchInput → engine.Match → CompareBenchmark. That is exactly the path a // real project WITHOUT ground truth takes, so it measures what actually ships. // // It runs every registered GT through the same code and prints per-GT plus a // side-by-side table, so a generic engine change can be checked against ALL // ground truths at once (no overfitting to a single machine). // ============================================================================ // gtCase describes one ground-truth benchmark fixture. type gtCase struct { name string path string machineType string // narrative is the machine description fed to ParseNarrative. We read it // from the GT JSON's machine_description field; if absent we fall back to // the GT's generic description. Authored narratives are intentionally NOT // keyword-stuffed — they represent how an engineer would describe the // machine, so the benchmark stays honest about extraction quality. narrativeOverride string } // gtBenchmarkCases is the registry the harness iterates over. Add a new GT // here and it is automatically cross-validated against every engine change. var gtBenchmarkCases = []gtCase{ { name: "Bremse (Roboterzelle)", path: "ground_truth_bremse.json", machineType: "robotics_cobot", narrativeOverride: "Automatisierte Roboterzelle zur Handhabung und Bearbeitung von " + "Bremsscheiben. Ein Industrieroboter mit Greifer entnimmt Bremsscheiben vom " + "Foerderband und legt sie in eine Bearbeitungsstation mit Drehtisch. Die Zelle ist " + "mit Schutzzaun, verriegelter Schutztuer und Lichtgitter gesichert. Antrieb ueber " + "Servomotoren und Frequenzumrichter, Steuerung ueber Sicherheits-SPS und Bedienpult. " + "Pneumatische Greifer und Spannvorrichtungen. Betrieb im Automatikbetrieb, Einrichten " + "und Einlernen (Teachen), Wartung und Stoerungsbeseitigung. Gefaehrdungen durch " + "Quetschen und Einzug bei Roboterbewegung, elektrische Energie und Druckluft.", }, { name: "Kistenhub (Hebevorrichtung)", path: "ground_truth_kistenhub.json", machineType: "lift", narrativeOverride: "Mobiles, fahrbares Kistenhubgeraet zum Heben und Positionieren von " + "Kisten und Lasten. Eine elektrisch angetriebene Hubplattform (Scherenhubtisch) hebt " + "die Last ueber ein Hubwerk. Antrieb ueber Elektromotor, Schaltschrank und Steuerung " + "mit Bedienpult. Das Geraet steht auf einem fahrbaren Fahrwerk mit Lenkrollen, daher " + "sind Standsicherheit und Kippgefahr relevant. Bediener heben Kisten manuell auf die " + "Plattform. Betrieb, manuelle Bedienung, Wartung, Reinigung und Transport. Elektrische " + "Gefaehrdungen durch Netzanschluss, Schaltschrank und Leitungen.", }, } // readGTNarrative extracts a machine narrative from the raw GT JSON, trying the // richer machine_description field before the generic description. func readGTNarrative(t *testing.T, path string) (gt GroundTruth, narrative, machineName string) { t.Helper() raw, err := os.ReadFile(filepath.Join("testdata", path)) if err != nil { t.Fatalf("read GT %s: %v", path, err) } if err := json.Unmarshal(raw, >); err != nil { t.Fatalf("parse GT %s: %v", path, err) } var extra struct { MachineName string `json:"machine_name"` MachineDescription string `json:"machine_description"` } _ = json.Unmarshal(raw, &extra) narrative = extra.MachineDescription if narrative == "" { narrative = gt.Description } return gt, narrative, extra.MachineName } // parseResultToMatchInput converts the deterministic narrative parse into the // engine's MatchInput, mirroring what the production handler does. func parseResultToMatchInput(pr ParseResult, machineType string) MatchInput { compIDs := make([]string, 0, len(pr.Components)) for _, c := range pr.Components { compIDs = append(compIDs, c.LibraryID) } energyIDs := make([]string, 0, len(pr.EnergySources)) for _, e := range pr.EnergySources { energyIDs = append(energyIDs, e.SourceID) } mt := []string{} if machineType != "" { mt = []string{machineType} } return MatchInput{ ComponentLibraryIDs: compIDs, EnergySourceIDs: energyIDs, LifecyclePhases: pr.LifecyclePhases, CustomTags: pr.CustomTags, OperationalStates: pr.OperationalStates, StateTransitions: pr.StateTransitions, HumanRoles: pr.Roles, MachineTypes: mt, } } // runGTCase runs the full narrative→measures pipeline for one GT and returns // the benchmark result plus the parse result for extraction-quality reporting. func runGTCase(t *testing.T, c gtCase) (*BenchmarkResult, ParseResult) { gt, narrative, _ := readGTNarrative(t, c.path) if c.narrativeOverride != "" { narrative = c.narrativeOverride } pr := ParseNarrative(narrative, c.machineType) input := parseResultToMatchInput(pr, c.machineType) engine := NewPatternEngine() out := engine.Match(input) hazards, mitigations := patternsToHazardsAndMitigations(out) return CompareBenchmark(>, hazards, mitigations), pr } // TestGT_RealNarrativeBenchmark runs every registered GT through the real // pipeline and prints a side-by-side comparison. Reporting only (no hard // thresholds yet) — run with: // // go test -v -vet=off -run TestGT_RealNarrativeBenchmark ./internal/iace/ func TestGT_RealNarrativeBenchmark(t *testing.T) { type row struct { name string comps, energy, tags int gtN, matched, extra int coverage, precision, measC float64 } var rows []row for _, c := range gtBenchmarkCases { res, pr := runGTCase(t, c) precision := 0.0 if res.TotalEngine > 0 { precision = float64(len(res.MatchedPairs)) / float64(res.TotalEngine) } rows = append(rows, row{ name: c.name, comps: len(pr.Components), energy: len(pr.EnergySources), tags: len(pr.CustomTags), gtN: res.TotalGT, matched: len(res.MatchedPairs), extra: len(res.ExtraInEngine), coverage: res.CoverageScore, precision: precision, measC: res.MeasureCoverage, }) t.Logf("=== %s (machine_type=%s) ===", c.name, c.machineType) t.Logf(" Narrative extraction: %d components, %d energy sources, %d custom tags", len(pr.Components), len(pr.EnergySources), len(pr.CustomTags)) t.Logf(" Coverage: %.1f%% (%d/%d) | Precision: %.1f%% | Measure: %.1f%% | Extras: %d", res.CoverageScore*100, len(res.MatchedPairs), res.TotalGT, precision*100, res.MeasureCoverage*100, len(res.ExtraInEngine)) sample := res.ExtraInEngine if len(sample) > 18 { sample = sample[:18] } t.Logf(" --- Extra-Sample (unmatched engine hazards) ---") for _, e := range sample { t.Logf(" [%s] %s", e.Category, abbrev(e.Name, 70)) } } t.Logf("\n=== Cross-GT summary (real narrative pipeline) ===") t.Logf(" %-28s %5s %5s %5s | %8s %9s %8s", "GT", "comp", "enrg", "tags", "coverage", "precision", "measure") for _, r := range rows { t.Logf(" %-28s %5d %5d %5d | %7.1f%% %8.1f%% %7.1f%%", r.name, r.comps, r.energy, r.tags, r.coverage*100, r.precision*100, r.measC*100) } // Regression guard: the real narrative pipeline (what ships for projects // without a GT) must keep high recall on both validated machines. const coverageFloor = 0.90 for _, r := range rows { if r.coverage < coverageFloor { t.Errorf("%s: real-pipeline coverage %.1f%% below floor %.0f%%", r.name, r.coverage*100, coverageFloor*100) } } } // foreignDomainTerms are machine-specific terms that betray a pattern's home // domain. If a pattern's own scenario/name contains one of these but the // pattern fires for an unrelated machine (a lift, a robot cell), it has leaked // across domains — the precision bug. Used to prioritise capability-domain // gating by real leak frequency, not guesswork. var foreignDomainTerms = map[string]string{ "spritzgie": "plastics", "extruder": "plastics", "kunststoffschmelze": "plastics", "spinnmaschine": "textile", "webmaschine": "textile", "spinnerei": "textile", "zweiwalzenwerk": "rolling", "walzwerk": "rolling", "kalander": "rolling", "gondel": "wind_lift", "pv-modul": "solar", "photovoltaik": "solar", "pv-anlage": "solar", "presse": "press", "schliesseinheit": "plastics", "drehmaschine": "cnc", "fraesmaschine": "cnc", "schleifscheibe": "grinding", "traktor": "agri", "harvester": "agri", "maehdrescher": "agri", "ballenpresse": "agri", "schweissen": "welding", "lichtbogenschweiss": "welding", "rolltreppe": "escalator", "fahrtreppe": "escalator", "spinnerei ": "textile", "extrusion": "plastics", } // TestGT_DomainLeakage names the patterns that leak across domains. For each GT // it runs the real pipeline, then flags every fired pattern whose own scenario // text references a foreign machine. The output is the prioritised gating list // for capability-domain hardening. // // go test -v -vet=off -run TestGT_DomainLeakage ./internal/iace/ func TestGT_DomainLeakage(t *testing.T) { leakCount := map[string]int{} // patternID → #GTs it leaked into leakInfo := map[string]string{} for _, c := range gtBenchmarkCases { _, narrative, _ := readGTNarrative(t, c.path) if c.narrativeOverride != "" { narrative = c.narrativeOverride } pr := ParseNarrative(narrative, c.machineType) out := NewPatternEngine().Match(parseResultToMatchInput(pr, c.machineType)) var leaks []string for _, pm := range out.MatchedPatterns { text := normalizeDE(pm.PatternName + " " + pm.ScenarioDE) for term, domain := range foreignDomainTerms { if strings.Contains(text, term) { leaks = append(leaks, pm.PatternID) leakCount[pm.PatternID]++ leakInfo[pm.PatternID] = domain + " :: " + abbrev(pm.ScenarioDE, 55) break } } } sort.Strings(leaks) t.Logf("=== %s (machine_type=%s): %d/%d fired patterns leaked from foreign domains ===", c.name, c.machineType, len(leaks), len(out.MatchedPatterns)) } type lk struct { id, info string n int } var all []lk for id, n := range leakCount { all = append(all, lk{id, leakInfo[id], n}) } sort.Slice(all, func(i, j int) bool { if all[i].n != all[j].n { return all[i].n > all[j].n } return all[i].id < all[j].id }) t.Logf("\n--- Leaking patterns (prioritised; n=#GTs affected) ---") t.Logf("Total distinct leaking patterns: %d", len(all)) for _, x := range all { t.Logf(" n=%d %-9s [%s]", x.n, x.id, x.info) } // Regression guard: no domain-specific pattern may fire for an unrelated // machine. A new leak means a pattern naming a foreign machine lacks its // domain capability gate (pattern_domain_gates.go). if len(all) > 0 { t.Errorf("cross-domain leakage must be 0; %d patterns leaked. "+ "Add the betraying term → domain tag in pattern_domain_gates.go (and emit it in keyword_dictionary.go).", len(all)) } }