feat(ai-sdk): searchControls — recall control sources on implementation questions (#39)

ai-compliance-sdk/internal/ucca/authority_rerank.go

@@ -120,21 +120,22 @@ func rerankByAuthority(query string, results []LegalSearchResult) []LegalSearchR
 	qForeign := queryIsForeign(query)
 	wantsGuidance := queryWantsGuidance(query)
 	wantsControls := queryWantsControls(query)
-	bestBindingSem := bestBindingSemantic(results, wantsGuidance || wantsControls)
+	bestBindingSem := bestBindingSemantic(results, wantsGuidance)
 
 	out := make([]LegalSearchResult, len(results))
 	copy(out, results)
 	for i := range out {
 		out[i].Score = authorityScore(query, out[i], qDomain, qForeign)
 	}
-	// Explicit interpretation intent → a competitive guideline may outrank binding;
-	// explicit implementation intent → a competitive technical_standard may. Both lift
-	// ABOVE the best binding FINAL, so a pure norm question (neither intent) is untouched.
+	// Explicit interpretation intent → a competitive guideline may outrank binding (lift
+	// above the best binding FINAL). Explicit implementation intent → boost the CONTROL-POOL
+	// (operational/procedural requirement, control standard, implementation guidance) over
+	// the abstract obligation, soft-ordered by role. Norm questions (neither) stay untouched.
 	if wantsGuidance {
 		liftAboveBinding(out, results, bestBindingSem, "supervisory_guidance")
 	}
 	if wantsControls {
-		liftAboveBinding(out, results, bestBindingSem, "technical_standard")
+		applyControlRoles(out)
 	}
 	sort.SliceStable(out, func(a, b int) bool {
 		return out[a].Score > out[b].Score

ai-compliance-sdk/internal/ucca/control_role.go

@@ -0,0 +1,123 @@
+package ucca
+
+import "strings"
+
+// source_role is the FUNCTIONAL role of a chunk — WHAT must be done (obligation),
+// HOW to implement it (operational/procedural requirement, control standard,
+// implementation guidance), or how to READ the norm (interpretation/definition).
+// It is ORTHOGONAL to source_class (legal authority): source_class decides RANK,
+// source_role decides CONTROL-POOL membership for implementation questions.
+// Derived deterministically from markers, so the untagged corpus needs no re-tag.
+const (
+	roleObligation      = "obligation"              // the abstract duty (the WHAT)
+	roleOperationalReq  = "operational_requirement" // concrete binding requirement (CRA Annex I)
+	roleProceduralReq   = "procedural_requirement"  // a process: notification/registration/DPIA/incident report
+	roleControlStandard = "control_standard"        // best-practice control catalog (NIST/OWASP/ISO/CIS)
+	roleImplGuidance    = "implementation_guidance" // advisory how-to (ENISA good practices, BSI)
+	roleInterpretation  = "interpretation"          // interprets the norm's MEANING (EDPB guideline)
+	roleDefinition      = "definition"              // definitions / scope / recitals
+)
+
+var (
+	proceduralMarkers = []string{
+		"Meldung", "Meldepflicht", "Notification", "Notifizierung", "Registrierung",
+		"Registration", "Konformitätserklärung", "Declaration of Conformity", "Incident",
+		"Berichterstattung", "Reporting", "Folgenabschätzung", "DSFA", "DPIA", "Anzeigepflicht",
+	}
+	annexMarkers       = []string{"Anhang", "Annex", "Appendix", "Anlage"}
+	operationalMarkers = []string{"Anforderung", "Requirement", "essential", "wesentliche"}
+	implMarkers        = []string{
+		"Good Practice", "Best Practice", "Standards Mapping", "Umsetzung", "Implementation",
+		"Handreichung", "Maßnahmenkatalog", "ICS", "SCADA", "Technical Guideline", "TIG",
+	}
+	definitionMarkers = []string{"Begriffsbestimmung", "Definition"}
+)
+
+// classifyRole derives the functional source_role from chunk metadata + the authority
+// class. technical_standard is always a control_standard; guidance splits into
+// implementation_guidance (how-to) vs interpretation (meaning); binding splits into
+// procedural / operational requirement / definition / plain obligation.
+func classifyRole(r LegalSearchResult) string {
+	cls := classifyAuthority(r).sourceClass
+	hay := strings.ToLower(r.ArticleLabel + " " + r.RegulationShort + " " + r.RegulationName + " " + r.Article)
+	switch {
+	case r.IsRecital:
+		return roleDefinition
+	case cls == "technical_standard":
+		return roleControlStandard
+	case cls == "supervisory_guidance":
+		if containsAnyLower(hay, implMarkers) {
+			return roleImplGuidance
+		}
+		return roleInterpretation
+	case cls == "binding_law":
+		switch {
+		case containsAnyLower(hay, definitionMarkers):
+			return roleDefinition
+		case containsAnyLower(hay, proceduralMarkers):
+			return roleProceduralReq
+		case containsAnyLower(hay, annexMarkers) || containsAnyLower(hay, operationalMarkers):
+			return roleOperationalReq
+		default:
+			return roleObligation
+		}
+	default:
+		return roleObligation
+	}
+}
+
+// controlRoleBonus is the soft intra-pool preference (User 2026-06-24):
+// operational_requirement > procedural_requirement > control_standard > implementation_guidance.
+var controlRoleBonus = map[string]float64{
+	roleOperationalReq:  0.100,
+	roleProceduralReq:   0.075,
+	roleControlStandard: 0.050,
+	roleImplGuidance:    0.000,
+}
+
+// controlPoolGain lifts EVERY control-pool role over the non-control roles (obligation/
+// interpretation/definition) on an implementation question, so the binding abstract
+// obligation does not dominate by authority alone. The obligation is not removed — it
+// stays visible as "Rechtsgrundlage" context below the recommended measures.
+const controlPoolGain = 0.15
+
+// applyControlRoles boosts the control-pool (the four implementation roles) for an
+// EXPLICIT implementation question, soft-ordered op_req > procedural > standard > guidance.
+// Replaces the earlier "lift technical_standard above binding" — controls are not only
+// technical_standard, and the binding operational_requirement (e.g. CRA Annex I) should win.
+func applyControlRoles(out []LegalSearchResult) {
+	for i := range out {
+		if bonus, ok := controlRoleBonus[classifyRole(out[i])]; ok {
+			out[i].Score += controlPoolGain + bonus
+		}
+	}
+}
+
+// isControlPoolRole reports whether a role belongs to the control-pool surfaced on
+// implementation questions (the four "how to implement" roles).
+func isControlPoolRole(role string) bool {
+	switch role {
+	case roleOperationalReq, roleProceduralReq, roleControlStandard, roleImplGuidance:
+		return true
+	}
+	return false
+}
+
+// controlRoleOf classifies a raw Qdrant payload into a source_role, so searchControls can
+// filter its deep dense pull to the control-pool BEFORE hits are mapped to LegalSearchResult.
+func controlRoleOf(payload map[string]interface{}) string {
+	article := getString(payload, "article")
+	if article == "" {
+		article = getString(payload, "section")
+	}
+	return classifyRole(LegalSearchResult{
+		RegulationShort: getString(payload, "regulation_short"),
+		RegulationName:  getString(payload, "regulation_name_de"),
+		ArticleLabel:    getString(payload, "article_label"),
+		Article:         article,
+		Category:        getString(payload, "category"),
+		SourceClass:     getString(payload, "source_class"),
+		AuthorityWeight: getInt(payload, "authority_weight"),
+		IsRecital:       getBool(payload, "is_recital"),
+	})
+}

ai-compliance-sdk/internal/ucca/control_role_test.go

@@ -0,0 +1,79 @@
+package ucca
+
+import "testing"
+
+func TestClassifyRole(t *testing.T) {
+	tests := []struct {
+		name string
+		r    LegalSearchResult
+		want string
+	}{
+		{"NIST -> control_standard", LegalSearchResult{RegulationShort: "NIST SP 800-82r3", ArticleLabel: "AU-8"}, roleControlStandard},
+		{"OWASP -> control_standard", LegalSearchResult{RegulationShort: "OWASP ASVS"}, roleControlStandard},
+		{"CRA Anhang -> operational_requirement", LegalSearchResult{RegulationShort: "CRA", ArticleLabel: "CRA Anhang I", Category: "regulation"}, roleOperationalReq},
+		{"CRA Meldepflicht -> procedural_requirement", LegalSearchResult{RegulationShort: "CRA", ArticleLabel: "Art. 14 CRA Meldepflicht", Category: "regulation"}, roleProceduralReq},
+		{"ENISA Good Practices -> implementation_guidance", LegalSearchResult{RegulationShort: "ENISA Supply Chain Good Practices"}, roleImplGuidance},
+		{"EDPB Leitlinie -> interpretation", LegalSearchResult{RegulationShort: "EDPB DPO", ArticleLabel: "WP243 Leitlinien Datenschutzbeauftragte"}, roleInterpretation},
+		{"DORA article -> obligation", LegalSearchResult{RegulationShort: "DORA", ArticleLabel: "Art. 5 DORA", Category: "regulation"}, roleObligation},
+		{"DSGVO Begriffsbestimmungen -> definition", LegalSearchResult{RegulationShort: "DSGVO", ArticleLabel: "Art. 4 DSGVO Begriffsbestimmungen", Category: "regulation"}, roleDefinition},
+		{"recital -> definition", LegalSearchResult{RegulationShort: "CRA", IsRecital: true}, roleDefinition},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := classifyRole(tt.r); got != tt.want {
+				t.Errorf("classifyRole() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestApplyControlRoles_PoolPreference(t *testing.T) {
+	// op_req > procedural > control_standard > impl_guidance; non-control roles get no boost.
+	roles := []struct {
+		r        LegalSearchResult
+		wantGain float64
+	}{
+		{LegalSearchResult{ArticleLabel: "CRA Anhang I", Category: "regulation"}, controlPoolGain + 0.100},
+		{LegalSearchResult{ArticleLabel: "Art. 14 CRA Meldepflicht", Category: "regulation"}, controlPoolGain + 0.075},
+		{LegalSearchResult{RegulationShort: "NIST SP 800-53"}, controlPoolGain + 0.050},
+		{LegalSearchResult{RegulationShort: "ENISA Good Practices"}, controlPoolGain + 0.000},
+		{LegalSearchResult{ArticleLabel: "Art. 5 DORA", Category: "regulation"}, 0.0}, // obligation: no boost
+	}
+	for _, rc := range roles {
+		out := []LegalSearchResult{rc.r}
+		out[0].Score = 1.0
+		applyControlRoles(out)
+		if got := out[0].Score - 1.0; got < rc.wantGain-1e-9 || got > rc.wantGain+1e-9 {
+			t.Errorf("role %q: gain %.3f, want %.3f", classifyRole(rc.r), got, rc.wantGain)
+		}
+	}
+}
+
+func TestIsControlPoolRole(t *testing.T) {
+	for _, r := range []string{roleOperationalReq, roleProceduralReq, roleControlStandard, roleImplGuidance} {
+		if !isControlPoolRole(r) {
+			t.Errorf("%q should be in the control-pool", r)
+		}
+	}
+	for _, r := range []string{roleObligation, roleInterpretation, roleDefinition} {
+		if isControlPoolRole(r) {
+			t.Errorf("%q should NOT be in the control-pool", r)
+		}
+	}
+}
+
+func TestControlRoleOf_Payload(t *testing.T) {
+	// searchControls filters its deep dense pull by classifying the raw Qdrant payload.
+	nist := map[string]interface{}{"regulation_short": "NIST SP 800-82r3", "article": "AU-8"}
+	if got := controlRoleOf(nist); got != roleControlStandard {
+		t.Errorf("untagged NIST payload role = %q, want control_standard", got)
+	}
+	craAnnex := map[string]interface{}{"regulation_short": "CRA", "article": "Anhang-I", "category": "regulation"}
+	if got := controlRoleOf(craAnnex); got != roleOperationalReq {
+		t.Errorf("CRA Anhang payload role = %q, want operational_requirement", got)
+	}
+	dora := map[string]interface{}{"regulation_short": "DORA", "article_label": "Art. 5 DORA", "category": "regulation"}
+	if got := controlRoleOf(dora); isControlPoolRole(got) {
+		t.Errorf("DORA abstract article role = %q must be excluded from the control-pool", got)
+	}
+}

ai-compliance-sdk/internal/ucca/legal_rag_client.go

@@ -107,6 +107,15 @@ func (c *LegalRAGClient) searchInternal(ctx context.Context, collection string,
 		hits = mergeDedupHits(hits, bindingHits)
 	}
 
+	// Control-Augmentation: bei expliziter Umsetzungsfrage einen tiefen dense-Pool ziehen und
+	// nur die Control-Pool-Rollen behalten — so werden NIST/CRA-Anhang (dense rank ~8-9, unter
+	// dem kleinen top-K) Kandidaten. Re-Rank/applyControlRoles ordnen sie danach.
+	if queryWantsControls(query) {
+		if controlHits, cErr := c.searchControls(ctx, collection, embedding); cErr == nil {
+			hits = mergeDedupHits(hits, controlHits)
+		}
+	}
+
 	// Graph-Augmentation: verbundene Normen (references_out/in) der Top-Hits ueber die
 	// praezise Zitations-Kante in den Pool ziehen — z.B. Art. 13 CRA zieht Anhang I (die
 	// eigentliche Pflichtquelle). Pool-Augmentation only; Re-Rank + topK bleiben.

ai-compliance-sdk/internal/ucca/legal_rag_http.go

@@ -204,6 +204,34 @@ func (c *LegalRAGClient) searchBinding(ctx context.Context, collection string, e
 	return c.doPointsSearch(ctx, collection, searchReq)
 }
 
+// controlPoolDepth is how deep the dense control pull reaches. Measured: for an EU-cyber
+// control query the relevant control sources sit at dense rank ~8-9 (NIST, CRA Annex), far
+// below the client's small top-K — so a fixed dense depth of 60 reliably surfaces them.
+const controlPoolDepth = 60
+
+// searchControls fetches a DEEP dense pool and keeps only the control-pool roles, so control
+// sources that the small top-K (hybrid) search misses become candidates on an implementation
+// question. Role is derived in code (no source_role tag needed). AUGMENTS the pool — the
+// caller gates it on control-intent.
+func (c *LegalRAGClient) searchControls(ctx context.Context, collection string, embedding []float64) ([]qdrantSearchHit, error) {
+	searchReq := qdrantSearchRequest{
+		Vector:      embedding,
+		Limit:       controlPoolDepth,
+		WithPayload: true,
+	}
+	hits, err := c.doPointsSearch(ctx, collection, searchReq)
+	if err != nil {
+		return nil, err
+	}
+	kept := make([]qdrantSearchHit, 0, len(hits))
+	for _, h := range hits {
+		if isControlPoolRole(controlRoleOf(h.Payload)) {
+			kept = append(kept, h)
+		}
+	}
+	return kept, nil
+}
+
 // doPointsSearch issues a POST /points/search and decodes the hits.
 func (c *LegalRAGClient) doPointsSearch(ctx context.Context, collection string, searchReq qdrantSearchRequest) ([]qdrantSearchHit, error) {
 	jsonBody, err := json.Marshal(searchReq)

ai-compliance-sdk/internal/ucca/legal_rag_intent_test.go

@@ -96,20 +96,21 @@ func TestQueryWantsControls(t *testing.T) {
 	}
 }
 
-func TestRerank_ControlQuestion_StandardMayWin(t *testing.T) {
-	// Explicit implementation intent + standard semantically competitive → standard wins.
+func TestRerank_ControlQuestion_OperationalReqTop(t *testing.T) {
+	// User priority for implementation questions: operational_requirement (binding concrete,
+	// CRA Anhang I) > control_standard (NIST). Both are in the control-pool; op_req wins.
 	results := []LegalSearchResult{
-		intentRes("NIST SP 800-82", "technical_standard", 0.62, 80),
-		intentRes("CRA", "binding_law", 0.58, 100),
+		{RegulationShort: "NIST SP 800-82r3", ArticleLabel: "AU-8", SourceClass: "technical_standard", AuthorityWeight: 80, Jurisdiction: "EU", Score: 0.60},
+		{RegulationShort: "CRA", ArticleLabel: "CRA Anhang I", Category: "regulation", Score: 0.58},
 	}
-	out := rerankByAuthority("Welche Controls passen zu Security Updates?", results)
-	if out[0].SourceClass != "technical_standard" {
-		t.Errorf("control question: technical_standard should win Top-1, got %s", out[0].SourceClass)
+	out := rerankByAuthority("Welche Controls und Massnahmen passen zu Security Updates?", results)
+	if out[0].RegulationShort != "CRA" {
+		t.Errorf("operational_requirement (CRA Anhang I) should be Top-1 over control_standard, got %q", out[0].RegulationShort)
 	}
 }
 
 func TestRerank_NormQuestion_BindingOverStandard(t *testing.T) {
-	// "Anforderungen" → no control intent → binding stays Top-1 over the standard.
+	// "Anforderungen" → no control intent → binding obligation stays Top-1 over the standard.
 	results := []LegalSearchResult{
 		intentRes("NIST SP 800-82", "technical_standard", 0.62, 80),
 		intentRes("CRA", "binding_law", 0.58, 100),
@@ -120,29 +121,15 @@ func TestRerank_NormQuestion_BindingOverStandard(t *testing.T) {
 	}
 }
 
-func TestRerank_OffTopicStandard_BlockedByGuard(t *testing.T) {
-	// Control intent present, but the standard is semantically far below binding →
-	// the margin guard keeps binding Top-1 (no off-topic standard override).
+func TestRerank_ControlQuestion_PoolBeatsBareObligation(t *testing.T) {
+	// A control-pool source (NIST control_standard) outranks an abstract obligation with no
+	// domain/topic advantage, because the implementation intent boosts the control-pool.
 	results := []LegalSearchResult{
-		intentRes("NIST SP 800-82", "technical_standard", 0.40, 80),
-		intentRes("CRA", "binding_law", 0.58, 100),
+		{RegulationShort: "NIST SP 800-82r3", ArticleLabel: "AU-8", SourceClass: "technical_standard", AuthorityWeight: 80, Jurisdiction: "EU", Score: 0.55},
+		{RegulationShort: "XYZ", ArticleLabel: "Art. 5 XYZ", Category: "regulation", Score: 0.58},
 	}
-	out := rerankByAuthority("Welche Controls passen zu Security Updates?", results)
-	if out[0].SourceClass != "binding_law" {
-		t.Errorf("off-topic standard must not win even with control intent, got %s", out[0].SourceClass)
-	}
-}
-
-func TestRerank_ControlQuestion_UntaggedNISTLifted(t *testing.T) {
-	// The existing NIST corpus is UNtagged (no source_class). It must still be classified
-	// technical_standard via markers and lifted on a control question — the whole reason
-	// the lift path classifies instead of trusting the raw payload field.
-	results := []LegalSearchResult{
-		{RegulationShort: "NIST SP 800-82r3", ArticleLabel: "AU-8", Score: 0.62},
-		{RegulationShort: "CRA", ArticleLabel: "Art. 13 CRA", Category: "regulation", Score: 0.58},
-	}
-	out := rerankByAuthority("Welche Controls passen zu Security Updates?", results)
+	out := rerankByAuthority("Welche Controls und Massnahmen passen zu Security Updates?", results)
 	if out[0].RegulationShort != "NIST SP 800-82r3" {
-		t.Errorf("untagged NIST should be lifted Top-1 on a control question, got %q", out[0].RegulationShort)
+		t.Errorf("control_standard should beat a bare abstract obligation on a control question, got %q", out[0].RegulationShort)
 	}
 }