feat(iace): NTRS harvester + licence gate (FMEA P2 stage 1)

Stage 1 of the FailureKnowledge bulk loader: harvest NASA NTRS lessons-learned with a strict public-reuse gate (NTRSUsable: public release, not export-controlled/EAR/ITAR, not CUI, PUBLIC_USE_PERMITTED, no third-party copyright). NTRSPDFURL prefers the PDF download for downstream text/OCR extraction. GET /iace/failure-knowledge/ntrs runs the live harvest and returns only the licence-clean records. Pure parse/gate helpers are fixture-tested (usable vs ITAR / third-party / restricted / video-only); accepted licences also pass the FK allowlist. Next: tuple extraction (abstract -> FailureKnowledge) + Playwright/OCR for scanned PDFs -> bp_iace_failure_kb. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-06-13 00:16:41 +02:00
parent 3f90e40807
commit d27c1b9e7d
4 changed files with 231 additions and 0 deletions
@@ -0,0 +1,66 @@
+package iace
+
+import "testing"
+
+// Fixture mirrors the real NTRS /search response shape (copyright is an OBJECT).
+const ntrsFixture = `{"results":[
+ {"id":20205010628,"title":"Lessons Learned from Large-Scale Aerospace Structural Testing","abstract":"A bracket fractured under load.",
+  "distribution":"PUBLIC","isLessonsLearned":true,
+  "copyright":{"determinationType":"GOV_PUBLIC_USE_PERMITTED","containsThirdPartyMaterial":false,"licenseType":"NO"},
+  "exportControl":{"isExportControl":"NO","ear":"NO","itar":"NO"},"cui":{"isCui":false},
+  "downloads":[{"mimetype":"application/pdf","links":{"original":"/api/citations/20205010628/downloads/paper.pdf"}}]},
+ {"id":1001,"title":"ITAR controlled","abstract":"x","distribution":"PUBLIC","isLessonsLearned":true,
+  "copyright":{"determinationType":"GOV_PUBLIC_USE_PERMITTED","containsThirdPartyMaterial":false},
+  "exportControl":{"isExportControl":"YES","ear":"NO","itar":"YES"},"cui":null,
+  "downloads":[{"mimetype":"application/pdf","links":{"original":"/x.pdf"}}]},
+ {"id":1002,"title":"Third party","abstract":"x","distribution":"PUBLIC","isLessonsLearned":true,
+  "copyright":{"determinationType":"PUBLIC_USE_PERMITTED","containsThirdPartyMaterial":true},
+  "exportControl":{"isExportControl":"NO"},"cui":null,"downloads":[]},
+ {"id":1003,"title":"Restricted dist","abstract":"x","distribution":"RESTRICTED",
+  "copyright":{"determinationType":"GOV_PUBLIC_USE_PERMITTED"},"exportControl":{"isExportControl":"NO"}},
+ {"id":1004,"title":"Video only","abstract":"x","distribution":"PUBLIC","isLessonsLearned":true,
+  "copyright":{"determinationType":"PUBLIC_USE_PERMITTED","containsThirdPartyMaterial":false},
+  "exportControl":{"isExportControl":"NO"},"cui":{"isCui":false},
+  "downloads":[{"mimetype":"video/mp4","links":{"original":"/v.mp4"}}]}
+]}`
+
+func TestParseNTRSSearch(t *testing.T) {
+	ls, err := parseNTRSSearch([]byte(ntrsFixture))
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+	if len(ls) != 5 {
+		t.Fatalf("expected 5 results, got %d", len(ls))
+	}
+	if ls[0].ID != 20205010628 || ls[0].Copyright.DeterminationType != "GOV_PUBLIC_USE_PERMITTED" {
+		t.Errorf("first record parsed wrong: %+v", ls[0])
+	}
+}
+
+func TestNTRSUsable_Gate(t *testing.T) {
+	ls, _ := parseNTRSSearch([]byte(ntrsFixture))
+	want := []bool{true, false, false, false, true} // ok, ITAR, third-party, restricted, ok(video)
+	for i, l := range ls {
+		ok, lic := NTRSUsable(l)
+		if ok != want[i] {
+			t.Errorf("record %d (%q): usable=%v, want %v", l.ID, l.Title, ok, want[i])
+		}
+		if ok && lic == "" {
+			t.Errorf("record %d usable but empty licence", l.ID)
+		}
+		// Every accepted record must also pass the failure-knowledge allowlist.
+		if ok && !FailureKnowledgeLicenseAllowed(lic) {
+			t.Errorf("record %d licence %q not allowed by FK allowlist", l.ID, lic)
+		}
+	}
+}
+
+func TestNTRSPDFURL(t *testing.T) {
+	ls, _ := parseNTRSSearch([]byte(ntrsFixture))
+	if got := NTRSPDFURL(ls[0]); got != "https://ntrs.nasa.gov/api/citations/20205010628/downloads/paper.pdf" {
+		t.Errorf("pdf url = %q", got)
+	}
+	if got := NTRSPDFURL(ls[4]); got != "" { // video-only → no PDF
+		t.Errorf("video-only should have no PDF url, got %q", got)
+	}
+}