Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a606000a20 | |||
| 6f0c1cf30d | |||
| f0120b237e | |||
| 1d65d99d5f | |||
| f2d445b891 | |||
| 08086ee75f | |||
| 1e5aaf7103 | |||
| af11d21f6e |
@@ -51,8 +51,8 @@ describe('advisor-rag', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('queryAdvisorRAG', () => {
|
describe('queryAdvisorRAG (Authority Router)', () => {
|
||||||
it('fragt alle 6 Collections ab und formatiert die Treffer', async () => {
|
it('ruft den Router EINMAL auf und formatiert die Treffer', async () => {
|
||||||
mockFetch.mockResolvedValue({
|
mockFetch.mockResolvedValue({
|
||||||
ok: true,
|
ok: true,
|
||||||
json: async () => ({ results: [{ text: 'Inhalt A', regulation_short: 'DSGVO', score: 0.9 }] }),
|
json: async () => ({ results: [{ text: 'Inhalt A', regulation_short: 'DSGVO', score: 0.9 }] }),
|
||||||
@@ -60,19 +60,19 @@ describe('advisor-rag', () => {
|
|||||||
const result = await mod.queryAdvisorRAG('Was ist eine DSFA?')
|
const result = await mod.queryAdvisorRAG('Was ist eine DSFA?')
|
||||||
expect(result).toContain('[Quelle 1: DSGVO]')
|
expect(result).toContain('[Quelle 1: DSGVO]')
|
||||||
expect(result).toContain('Inhalt A')
|
expect(result).toContain('Inhalt A')
|
||||||
expect(mockFetch).toHaveBeenCalledTimes(mod.COMPLIANCE_COLLECTIONS.length)
|
expect(mockFetch).toHaveBeenCalledTimes(1)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('ruft die ai-sdk /sdk/v1/rag/search mit collection + top_k auf', async () => {
|
it('ruft /sdk/v1/rag/retrieve mit query + top_k (ohne collection) auf', async () => {
|
||||||
mockFetch.mockResolvedValue({ ok: true, json: async () => ({ results: [] }) })
|
mockFetch.mockResolvedValue({ ok: true, json: async () => ({ results: [] }) })
|
||||||
await mod.queryAdvisorRAG('test')
|
await mod.queryAdvisorRAG('test')
|
||||||
expect(mockFetch).toHaveBeenCalledWith(
|
expect(mockFetch).toHaveBeenCalledWith(
|
||||||
expect.stringContaining('/sdk/v1/rag/search'),
|
expect.stringContaining('/sdk/v1/rag/retrieve'),
|
||||||
expect.objectContaining({ method: 'POST' }),
|
expect.objectContaining({ method: 'POST' }),
|
||||||
)
|
)
|
||||||
const body = JSON.parse(mockFetch.mock.calls[0][1].body)
|
const body = JSON.parse(mockFetch.mock.calls[0][1].body)
|
||||||
expect(body).toMatchObject({ query: 'test', top_k: 3 })
|
expect(body).toMatchObject({ query: 'test', top_k: 8 })
|
||||||
expect(mod.COMPLIANCE_COLLECTIONS).toContain(body.collection)
|
expect(body.collection).toBeUndefined()
|
||||||
})
|
})
|
||||||
|
|
||||||
it('liefert leeren String wenn das RAG-Backend nicht erreichbar ist (graceful)', async () => {
|
it('liefert leeren String wenn das RAG-Backend nicht erreichbar ist (graceful)', async () => {
|
||||||
@@ -80,10 +80,5 @@ describe('advisor-rag', () => {
|
|||||||
const result = await mod.queryAdvisorRAG('test')
|
const result = await mod.queryAdvisorRAG('test')
|
||||||
expect(result).toBe('')
|
expect(result).toBe('')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('umfasst genau die 6 Compliance-Collections', () => {
|
|
||||||
expect(mod.COMPLIANCE_COLLECTIONS).toHaveLength(6)
|
|
||||||
expect(mod.COMPLIANCE_COLLECTIONS).toContain('bp_compliance_recht')
|
|
||||||
})
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
/**
|
/**
|
||||||
* Compliance-Advisor RAG-Suche.
|
* Compliance-Advisor RAG-Suche.
|
||||||
*
|
*
|
||||||
* Fragt die ai-compliance-sdk (`/sdk/v1/rag/search`) ab statt des frueheren
|
* Fragt den Authority Router der ai-compliance-sdk (`/sdk/v1/rag/retrieve`) mit NUR der
|
||||||
* `rag-service:8097` (auf prod nicht erreichbar). Die ai-sdk embeddet die Query
|
* Query ab — der Router waehlt selbst die Collections (Broad-Authority-Base + KB-2026.1-Slice
|
||||||
* mit bge-m3 (prod: ollama-embed) und sucht in den Qdrant-Compliance-Collections
|
* bei in-scope), embeddet mit bge-m3 (prod: ollama-embed), merged + authority-ranked. Der
|
||||||
* — damit profitiert der Advisor vom reicheren Embedding.
|
* Advisor bleibt damit collection-agnostisch (Vertrag: Compiler -> Collections -> Retriever
|
||||||
|
* -> Advisor); die fruehere Multi-Collection-Logik liegt jetzt im Retriever.
|
||||||
*
|
*
|
||||||
* Fehler je Collection werden geschluckt (graceful: Antwort ohne diesen Treffer).
|
* Fehler werden geschluckt (graceful: Antwort ohne RAG-Kontext).
|
||||||
* Fundstellen via article_label sind live ab dem Prod-Re-Ingest 2026-06.
|
* Fundstellen via article_label sind live ab dem Prod-Re-Ingest 2026-06.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
@@ -17,16 +18,6 @@ const DEFAULT_USER = '00000000-0000-0000-0000-000000000001'
|
|||||||
const DEFAULT_TENANT =
|
const DEFAULT_TENANT =
|
||||||
process.env.DEFAULT_TENANT_ID || '9282a473-5c95-4b3a-bf78-0ecc0ec71d3e'
|
process.env.DEFAULT_TENANT_ID || '9282a473-5c95-4b3a-bf78-0ecc0ec71d3e'
|
||||||
|
|
||||||
// Compliance-relevante Collections (ai-sdk-Whitelist `AllowedCollections`).
|
|
||||||
export const COMPLIANCE_COLLECTIONS = [
|
|
||||||
'bp_compliance_gesetze',
|
|
||||||
'bp_compliance_ce',
|
|
||||||
'bp_compliance_datenschutz',
|
|
||||||
'bp_dsfa_corpus',
|
|
||||||
'bp_compliance_recht',
|
|
||||||
'bp_legal_templates',
|
|
||||||
] as const
|
|
||||||
|
|
||||||
interface SdkRagResult {
|
interface SdkRagResult {
|
||||||
text?: string
|
text?: string
|
||||||
regulation_code?: string
|
regulation_code?: string
|
||||||
@@ -68,39 +59,36 @@ export function mapSdkResults(results: SdkRagResult[] | undefined): ScoredPassag
|
|||||||
.filter((p) => p.content)
|
.filter((p) => p.content)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function searchCollection(collection: string, query: string): Promise<ScoredPassage[]> {
|
/**
|
||||||
|
* Authority Router: EIN collection-agnostischer Aufruf an die ai-sdk (`/sdk/v1/rag/retrieve`).
|
||||||
|
* Der Router waehlt die Collections (Broad-Authority-Base + KB-2026.1-Slice bei in-scope),
|
||||||
|
* merged + authority-ranked sie und liefert die Top-Passagen. Der Advisor weiss damit nichts
|
||||||
|
* mehr ueber einzelne Collections — die fruehere Multi-Collection-Logik liegt jetzt im Retriever.
|
||||||
|
* Fehler werden geschluckt (graceful: Antwort ohne RAG-Kontext).
|
||||||
|
*/
|
||||||
|
export async function queryAdvisorRAG(query: string): Promise<string> {
|
||||||
|
let passages: ScoredPassage[] = []
|
||||||
try {
|
try {
|
||||||
const res = await fetch(`${SDK_URL}/sdk/v1/rag/search`, {
|
const res = await fetch(`${SDK_URL}/sdk/v1/rag/retrieve`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
'X-User-ID': DEFAULT_USER,
|
'X-User-ID': DEFAULT_USER,
|
||||||
'X-Tenant-ID': DEFAULT_TENANT,
|
'X-Tenant-ID': DEFAULT_TENANT,
|
||||||
},
|
},
|
||||||
body: JSON.stringify({ query, collection, top_k: 3 }),
|
body: JSON.stringify({ query, top_k: 8 }),
|
||||||
signal: AbortSignal.timeout(10000),
|
signal: AbortSignal.timeout(15000),
|
||||||
})
|
})
|
||||||
if (!res.ok) return []
|
if (res.ok) {
|
||||||
const data = await res.json()
|
const data = await res.json()
|
||||||
return mapSdkResults(data.results)
|
passages = mapSdkResults(data.results)
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
return []
|
// graceful: keine Verbindung -> Antwort ohne RAG-Kontext
|
||||||
}
|
}
|
||||||
}
|
// Der Router liefert bereits authority-geordnete Top-K; Reihenfolge bewahren.
|
||||||
|
if (passages.length === 0) return ''
|
||||||
/**
|
return passages
|
||||||
* Fragt alle Compliance-Collections parallel ab und liefert die Top-8-Passagen
|
|
||||||
* als formatierten Kontextblock (oder '' wenn nichts erreichbar/gefunden).
|
|
||||||
*/
|
|
||||||
export async function queryAdvisorRAG(query: string): Promise<string> {
|
|
||||||
const settled = await Promise.all(
|
|
||||||
COMPLIANCE_COLLECTIONS.map((c) => searchCollection(c, query)),
|
|
||||||
)
|
|
||||||
const all = settled.flat()
|
|
||||||
if (all.length === 0) return ''
|
|
||||||
all.sort((a, b) => b.score - a.score)
|
|
||||||
return all
|
|
||||||
.slice(0, 8)
|
|
||||||
.map((r, i) => `[Quelle ${i + 1}: ${r.source}]\n${r.content}`)
|
.map((r, i) => `[Quelle ${i + 1}: ${r.source}]\n${r.content}`)
|
||||||
.join('\n\n---\n\n')
|
.join('\n\n---\n\n')
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -82,6 +82,104 @@ func (h *RAGHandlers) Search(c *gin.Context) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RetrieveRequest is the Authority Router request: a query only, no collection — the router decides
|
||||||
|
// which collections to query (broad authority base + the in-scope KB-2026.1 slice).
|
||||||
|
type RetrieveRequest struct {
|
||||||
|
Query string `json:"query" binding:"required"`
|
||||||
|
TopK int `json:"top_k,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retrieve is the Authority Router endpoint. The Advisor calls this with ONLY a query and stays
|
||||||
|
// collection-agnostic; the router fans out over the authority base + the in-scope slice, merges by
|
||||||
|
// authority score, and returns the unified top-K. Response shape matches Search (query/results/
|
||||||
|
// count/assessment) so existing consumers parse it unchanged.
|
||||||
|
// POST /sdk/v1/rag/retrieve
|
||||||
|
func (h *RAGHandlers) Retrieve(c *gin.Context) {
|
||||||
|
var req RetrieveRequest
|
||||||
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.TopK <= 0 || req.TopK > 20 {
|
||||||
|
req.TopK = 8
|
||||||
|
}
|
||||||
|
|
||||||
|
results, err := h.ragClient.Retrieve(c.Request.Context(), req.Query, req.TopK)
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "RAG retrieve failed: " + err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evidence-Type-Schicht: die autoritative typisierte Evidence (Fußnoten/Tabellen/Figuren) aus
|
||||||
|
// dem KB-Wissensraum SEPARAT surfacen, statt sie im Breit-Basis-Text-Merge zu verlieren.
|
||||||
|
// results[] bleibt der Text-Kontext fürs LLM + die Quellen-Liste.
|
||||||
|
ev := h.ragClient.RetrieveEvidence(c.Request.Context(), req.Query)
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, gin.H{
|
||||||
|
"query": req.Query,
|
||||||
|
"results": results,
|
||||||
|
"count": len(results),
|
||||||
|
"assessment": ucca.Assess(results),
|
||||||
|
"footnotes": footnotesFromEvidence(ev[ucca.EvidenceFootnote]),
|
||||||
|
"tables": tablesFromEvidence(ev[ucca.EvidenceTable]),
|
||||||
|
"figures": figuresFromEvidence(ev[ucca.EvidenceFigure]),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// footnotesFromEvidence maps FOOTNOTE evidence to the Evidence-Workspace RawFootnote shape.
|
||||||
|
func footnotesFromEvidence(rs []ucca.LegalSearchResult) []gin.H {
|
||||||
|
out := make([]gin.H, 0, len(rs))
|
||||||
|
for _, r := range rs {
|
||||||
|
out = append(out, gin.H{
|
||||||
|
"id": r.CitationUnit,
|
||||||
|
"ref": r.CitationUnit,
|
||||||
|
"number": r.FootnoteLabel,
|
||||||
|
"regulation_code": r.RegulationCode,
|
||||||
|
"regulation_short": r.RegulationShort,
|
||||||
|
"regulation_name": r.RegulationName,
|
||||||
|
"section": r.RefCitationUnit,
|
||||||
|
"text": r.FootnoteVerbatim,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// tablesFromEvidence maps TABLE evidence (C6/C9). Key is present so the same Evidence-Type path
|
||||||
|
// carries tables the moment the UI adds a table section.
|
||||||
|
func tablesFromEvidence(rs []ucca.LegalSearchResult) []gin.H {
|
||||||
|
out := make([]gin.H, 0, len(rs))
|
||||||
|
for _, r := range rs {
|
||||||
|
out = append(out, gin.H{
|
||||||
|
"id": r.CitationUnit,
|
||||||
|
"caption": r.ArticleLabel,
|
||||||
|
"regulation_code": r.RegulationCode,
|
||||||
|
"regulation_short": r.RegulationShort,
|
||||||
|
"regulation_name": r.RegulationName,
|
||||||
|
"section": r.RefCitationUnit,
|
||||||
|
"text": r.Text,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// figuresFromEvidence maps FIGURE evidence (C8). Empty until C8 populates figure units; image_url/
|
||||||
|
// caption/vision_summary get added here when C8 lands — same path, no router change.
|
||||||
|
func figuresFromEvidence(rs []ucca.LegalSearchResult) []gin.H {
|
||||||
|
out := make([]gin.H, 0, len(rs))
|
||||||
|
for _, r := range rs {
|
||||||
|
out = append(out, gin.H{
|
||||||
|
"figure_id": r.CitationUnit,
|
||||||
|
"caption": r.ArticleLabel,
|
||||||
|
"regulation_code": r.RegulationCode,
|
||||||
|
"regulation_short": r.RegulationShort,
|
||||||
|
"regulation_name": r.RegulationName,
|
||||||
|
"section": r.RefCitationUnit,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// ListRegulations returns the list of available regulations in the corpus.
|
// ListRegulations returns the list of available regulations in the corpus.
|
||||||
// GET /sdk/v1/rag/regulations
|
// GET /sdk/v1/rag/regulations
|
||||||
func (h *RAGHandlers) ListRegulations(c *gin.Context) {
|
func (h *RAGHandlers) ListRegulations(c *gin.Context) {
|
||||||
|
|||||||
@@ -159,6 +159,7 @@ func registerRAGRoutes(v1 *gin.RouterGroup, h *handlers.RAGHandlers) {
|
|||||||
ragRoutes := v1.Group("/rag")
|
ragRoutes := v1.Group("/rag")
|
||||||
{
|
{
|
||||||
ragRoutes.POST("/search", h.Search)
|
ragRoutes.POST("/search", h.Search)
|
||||||
|
ragRoutes.POST("/retrieve", h.Retrieve)
|
||||||
ragRoutes.GET("/regulations", h.ListRegulations)
|
ragRoutes.GET("/regulations", h.ListRegulations)
|
||||||
ragRoutes.GET("/corpus-status", h.CorpusStatus)
|
ragRoutes.GET("/corpus-status", h.CorpusStatus)
|
||||||
ragRoutes.GET("/corpus-versions/:collection", h.CorpusVersionHistory)
|
ragRoutes.GET("/corpus-versions/:collection", h.CorpusVersionHistory)
|
||||||
@@ -358,7 +359,6 @@ func registerWhistleblowerRoutes(v1 *gin.RouterGroup, h *handlers.WhistleblowerH
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
func registerMaximizerRoutes(v1 *gin.RouterGroup, h *handlers.MaximizerHandlers) {
|
func registerMaximizerRoutes(v1 *gin.RouterGroup, h *handlers.MaximizerHandlers) {
|
||||||
m := v1.Group("/maximizer")
|
m := v1.Group("/maximizer")
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package ucca
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// routerBaseCollections is the broad authority base the Authority Router fans out over. It mirrors
|
||||||
|
// the Advisor's historical multi-collection set; the KB-2026.1 slice is added separately when the
|
||||||
|
// query is in scope. Override via RAG_ROUTER_COLLECTIONS (comma-separated) per environment.
|
||||||
|
func (c *LegalRAGClient) routerBaseCollections() []string {
|
||||||
|
if v := strings.TrimSpace(os.Getenv("RAG_ROUTER_COLLECTIONS")); v != "" {
|
||||||
|
var out []string
|
||||||
|
for _, p := range strings.Split(v, ",") {
|
||||||
|
if s := strings.TrimSpace(p); s != "" {
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(out) > 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return []string{
|
||||||
|
"bp_compliance_gesetze",
|
||||||
|
"bp_compliance_ce",
|
||||||
|
"bp_compliance_datenschutz",
|
||||||
|
"bp_dsfa_corpus",
|
||||||
|
"bp_compliance_recht",
|
||||||
|
"bp_legal_templates",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const routerPerCollectionTopK = 3
|
||||||
|
|
||||||
|
// Retrieve is the Authority Router entry point: callers (the Advisor) pass ONLY a query and stay
|
||||||
|
// collection-agnostic. The router fans out over the broad authority base and ADDS the KB-2026.1
|
||||||
|
// slice when the query is in scope (inKBScope), then merges all hits, deduplicates, and returns the
|
||||||
|
// top-K by authority score. This moves the former Advisor-side collection fan-out into the retrieval
|
||||||
|
// layer (the "Retriever" tier of the quality pyramid), so the proven KB-2026.1 slice gain reaches
|
||||||
|
// the product path without the Advisor knowing about individual collections.
|
||||||
|
//
|
||||||
|
// The merged set is ordered by the per-collection authority score that rerankByAuthority already
|
||||||
|
// produced inside searchInternal — i.e. binding-vs-guidance ordering is preserved across the merge.
|
||||||
|
// Per-collection failures (e.g. a collection absent on an environment) degrade gracefully.
|
||||||
|
func (c *LegalRAGClient) Retrieve(ctx context.Context, query string, topK int) ([]LegalSearchResult, error) {
|
||||||
|
if topK <= 0 {
|
||||||
|
topK = 8
|
||||||
|
}
|
||||||
|
|
||||||
|
collections := c.routerBaseCollections()
|
||||||
|
if c.kbScopeRoutingEnabled && c.kbSliceCollection != "" && inKBScope(query) {
|
||||||
|
collections = append(collections, c.kbSliceCollection)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cross-regulation queries (>=2 explicitly named regulations) get a larger per-collection budget
|
||||||
|
// so each collection's multi-regulation search isn't truncated down to the keyword-dominant
|
||||||
|
// domain; the final per-regulation balancing then guarantees every named domain in the top-K.
|
||||||
|
regs := detectRegulations(query)
|
||||||
|
perColl := routerPerCollectionTopK
|
||||||
|
if len(regs) >= 2 {
|
||||||
|
perColl = routerPerCollectionTopK * len(regs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warm the full-text indexes sequentially first so the concurrent fan-out below only READS the
|
||||||
|
// shared textIndexEnsured map (the writes happen here, serialized) — closes the cold-start map
|
||||||
|
// race deterministically. Best-effort: a missing collection just stays un-indexed (hybrid then
|
||||||
|
// falls back to dense, or the per-collection search degrades to nothing).
|
||||||
|
if c.hybridEnabled {
|
||||||
|
for _, coll := range collections {
|
||||||
|
_ = c.ensureTextIndex(ctx, coll)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([][]LegalSearchResult, len(collections))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i, coll := range collections {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, coll string) {
|
||||||
|
defer wg.Done()
|
||||||
|
if res, err := c.searchInternal(ctx, coll, query, nil, perColl); err == nil {
|
||||||
|
out[i] = res
|
||||||
|
}
|
||||||
|
}(i, coll)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
merged := make([]LegalSearchResult, 0, len(collections)*perColl)
|
||||||
|
for _, r := range out {
|
||||||
|
merged = append(merged, r...)
|
||||||
|
}
|
||||||
|
merged = dedupResults(merged)
|
||||||
|
sort.SliceStable(merged, func(a, b int) bool { return merged[a].Score > merged[b].Score })
|
||||||
|
|
||||||
|
// Cross-regulation: guarantee every named domain is represented (0070-class fix) instead of
|
||||||
|
// letting a global score-sort starve the non-dominant domain.
|
||||||
|
if len(regs) >= 2 {
|
||||||
|
return balanceByRegulation(merged, regs, topK), nil
|
||||||
|
}
|
||||||
|
if len(merged) > topK {
|
||||||
|
merged = merged[:topK]
|
||||||
|
}
|
||||||
|
return merged, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// dedupResults removes duplicate passages that can appear when collections overlap, keeping the
|
||||||
|
// highest-scoring occurrence. Identity = regulation_code + article_label + a text prefix.
|
||||||
|
func dedupResults(in []LegalSearchResult) []LegalSearchResult {
|
||||||
|
pos := make(map[string]int, len(in))
|
||||||
|
out := make([]LegalSearchResult, 0, len(in))
|
||||||
|
for _, r := range in {
|
||||||
|
text := r.Text
|
||||||
|
if len(text) > 80 {
|
||||||
|
text = text[:80]
|
||||||
|
}
|
||||||
|
key := r.RegulationCode + "|" + r.ArticleLabel + "|" + text
|
||||||
|
if idx, ok := pos[key]; ok {
|
||||||
|
if r.Score > out[idx].Score {
|
||||||
|
out[idx] = r
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pos[key] = len(out)
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package ucca
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type benchQ struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Document string `json:"document"`
|
||||||
|
Question string `json:"question"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// docTokens maps a bench question's expected document to acceptable regulation_code/label substrings.
|
||||||
|
func docTokens(document string) []string {
|
||||||
|
d := strings.ToUpper(document)
|
||||||
|
var t []string
|
||||||
|
for _, wp := range []string{"WP243", "WP248", "WP260"} {
|
||||||
|
if strings.Contains(d, wp) {
|
||||||
|
t = append(t, wp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dns := strings.ReplaceAll(d, " ", "")
|
||||||
|
for _, gl := range []struct{ key, tok string }{{"07/2020", "GL07"}, {"05/2020", "GL05"}, {"09/2022", "GL09"}} {
|
||||||
|
if strings.Contains(dns, gl.key) {
|
||||||
|
t = append(t, gl.tok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(d, "TDDDG") {
|
||||||
|
t = append(t, "TDDDG")
|
||||||
|
}
|
||||||
|
if strings.Contains(d, "DSGVO") || strings.Contains(d, "ART. 13") || strings.Contains(d, "ART. 14") {
|
||||||
|
t = append(t, "DSGVO")
|
||||||
|
}
|
||||||
|
if strings.Contains(d, "BDSG") {
|
||||||
|
t = append(t, "BDSG")
|
||||||
|
}
|
||||||
|
if strings.Contains(d, "CRA") {
|
||||||
|
t = append(t, "CRA")
|
||||||
|
}
|
||||||
|
if strings.Contains(d, "MASCH") {
|
||||||
|
t = append(t, "MASCH", "MACHINERY", "MVO")
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
func hitDoc(results []LegalSearchResult, toks []string) bool {
|
||||||
|
for _, r := range results {
|
||||||
|
s := strings.ReplaceAll(strings.ToUpper(r.RegulationCode+" "+r.ArticleLabel), " ", "")
|
||||||
|
for _, tk := range toks {
|
||||||
|
if strings.Contains(s, strings.ReplaceAll(tk, " ", "")) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMultiReg0070E2E (RUN_E2E=1) is the 0070 regression: a cross-regulation query (CRA + MaschVO)
|
||||||
|
// must return BOTH domains through the real Retrieve(), not just the keyword-dominant CRA.
|
||||||
|
func TestMultiReg0070E2E(t *testing.T) {
|
||||||
|
if os.Getenv("RUN_E2E") != "1" {
|
||||||
|
t.Skip("set RUN_E2E=1 + QDRANT_URL/OLLAMA_URL/QDRANT_API_KEY")
|
||||||
|
}
|
||||||
|
c := NewLegalRAGClient()
|
||||||
|
q := "Wie greifen CRA und Maschinenverordnung bei einer vernetzten Maschine ineinander?"
|
||||||
|
res, err := c.Retrieve(context.Background(), q, 8)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("retrieve: %v", err)
|
||||||
|
}
|
||||||
|
var hasCRA, hasMasch bool
|
||||||
|
var codes []string
|
||||||
|
for _, r := range res {
|
||||||
|
u := strings.ToUpper(r.RegulationCode)
|
||||||
|
codes = append(codes, u)
|
||||||
|
if strings.Contains(u, "CRA") {
|
||||||
|
hasCRA = true
|
||||||
|
}
|
||||||
|
if strings.Contains(u, "MASCH") || strings.Contains(u, "MACHIN") || u == "MVO" {
|
||||||
|
hasMasch = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Logf("0070 top-8 codes: %v", codes)
|
||||||
|
if !hasCRA || !hasMasch {
|
||||||
|
t.Errorf("0070 must return BOTH domains via Retrieve(): CRA=%v MaschVO=%v", hasCRA, hasMasch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAuthorityRouterCB100 (RUN_E2E=1) drives the REAL Retrieve() over the ComplianceBench-100 against
|
||||||
|
// the live collections: NEW (scope routing on → slice added for in-scope queries) vs OLD (routing off
|
||||||
|
// → broad base only). It is the regression gate that the router actually delivers the proven slice
|
||||||
|
// gain (+28/0-regr in the offline simulation) through the production Go code path.
|
||||||
|
func TestAuthorityRouterCB100(t *testing.T) {
|
||||||
|
if os.Getenv("RUN_E2E") != "1" {
|
||||||
|
t.Skip("set RUN_E2E=1 + QDRANT_URL/OLLAMA_URL/QDRANT_API_KEY + BENCH_PATH")
|
||||||
|
}
|
||||||
|
path := os.Getenv("BENCH_PATH")
|
||||||
|
if path == "" {
|
||||||
|
path = "/tmp/compliance_bench.json"
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("bench read: %v", err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Questions []benchQ `json:"questions"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||||
|
t.Fatalf("bench parse: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BENCH_STRIDE samples every Kth question (stratified across DS/CRA/MaschVO) so the gate stays
|
||||||
|
// tractable against the remote dev Qdrant; default 1 = full CB-100.
|
||||||
|
stride := 1
|
||||||
|
if s := os.Getenv("BENCH_STRIDE"); s != "" {
|
||||||
|
if n, err := strconv.Atoi(s); err == nil && n > 0 {
|
||||||
|
stride = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c := NewLegalRAGClient()
|
||||||
|
ctx := context.Background()
|
||||||
|
var n, oldHit, newHit, gain, regr int
|
||||||
|
for i, q := range doc.Questions {
|
||||||
|
if i%stride != 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
toks := docTokens(q.Document)
|
||||||
|
c.kbScopeRoutingEnabled = false
|
||||||
|
oldRes, _ := c.Retrieve(ctx, q.Question, 8)
|
||||||
|
c.kbScopeRoutingEnabled = true
|
||||||
|
newRes, _ := c.Retrieve(ctx, q.Question, 8)
|
||||||
|
oh, nh := hitDoc(oldRes, toks), hitDoc(newRes, toks)
|
||||||
|
if oh {
|
||||||
|
oldHit++
|
||||||
|
}
|
||||||
|
if nh {
|
||||||
|
newHit++
|
||||||
|
}
|
||||||
|
flip := "="
|
||||||
|
switch {
|
||||||
|
case !oh && nh:
|
||||||
|
gain++
|
||||||
|
flip = "GAIN"
|
||||||
|
case oh && !nh:
|
||||||
|
regr++
|
||||||
|
flip = "REGR"
|
||||||
|
}
|
||||||
|
t.Logf("%-9s [%-14s] OLD=%-5v NEW=%-5v %s", q.ID, q.Document, oh, nh, flip)
|
||||||
|
}
|
||||||
|
t.Logf("CB-100 sample (stride=%d) via Retrieve(): N=%d | OLD-hit %d | NEW-hit %d | GAIN %d | REGR %d",
|
||||||
|
stride, n, oldHit, newHit, gain, regr)
|
||||||
|
if newHit <= oldHit || gain < 3 {
|
||||||
|
t.Errorf("router must add slice gains: NEW(%d) must exceed OLD(%d), gain=%d", newHit, oldHit, gain)
|
||||||
|
}
|
||||||
|
if regr > 2 {
|
||||||
|
t.Errorf("too many regressions through the router: %d", regr)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package ucca
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRouterBaseCollections(t *testing.T) {
|
||||||
|
c := &LegalRAGClient{}
|
||||||
|
os.Unsetenv("RAG_ROUTER_COLLECTIONS")
|
||||||
|
def := c.routerBaseCollections()
|
||||||
|
if len(def) != 6 || def[1] != "bp_compliance_ce" {
|
||||||
|
t.Fatalf("default base collections unexpected: %v", def)
|
||||||
|
}
|
||||||
|
|
||||||
|
os.Setenv("RAG_ROUTER_COLLECTIONS", " bp_compliance_ce , kb_2026_1_build ,, ")
|
||||||
|
defer os.Unsetenv("RAG_ROUTER_COLLECTIONS")
|
||||||
|
got := c.routerBaseCollections()
|
||||||
|
if len(got) != 2 || got[0] != "bp_compliance_ce" || got[1] != "kb_2026_1_build" {
|
||||||
|
t.Fatalf("env override parse failed (trim/empty): %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRouterSliceSelection(t *testing.T) {
|
||||||
|
// The router appends the slice exactly when the query is in scope (inKBScope) and routing is on.
|
||||||
|
// Mirror the selection logic so a regression in either is caught without a live Qdrant.
|
||||||
|
c := &LegalRAGClient{kbSliceCollection: "kb_2026_1_build", kbScopeRoutingEnabled: true}
|
||||||
|
sel := func(q string) bool {
|
||||||
|
colls := c.routerBaseCollections()
|
||||||
|
if c.kbScopeRoutingEnabled && c.kbSliceCollection != "" && inKBScope(q) {
|
||||||
|
colls = append(colls, c.kbSliceCollection)
|
||||||
|
}
|
||||||
|
for _, x := range colls {
|
||||||
|
if x == c.kbSliceCollection {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !sel("Welche neun Kriterien nennt WP248 fuer ein hohes Risiko?") {
|
||||||
|
t.Error("in-scope guidance query must include the slice")
|
||||||
|
}
|
||||||
|
if sel("Was sagt NIST SP 800-53 zu Access Control?") {
|
||||||
|
t.Error("out-of-scope query must NOT include the slice")
|
||||||
|
}
|
||||||
|
c.kbScopeRoutingEnabled = false
|
||||||
|
if sel("Welche Kriterien nennt WP248?") {
|
||||||
|
t.Error("routing disabled => slice never included")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBalanceByRegulation(t *testing.T) {
|
||||||
|
regs := []detectedRegulation{
|
||||||
|
{Canonical: "CRA", CodeValues: []string{"CRA"}},
|
||||||
|
{Canonical: "MaschVO", CodeValues: []string{"MASCHVO", "MVO", "MACHINERY"}},
|
||||||
|
}
|
||||||
|
// CRA dominates by score; without balancing the top-4 would be all CRA + NIST.
|
||||||
|
pool := []LegalSearchResult{
|
||||||
|
{RegulationCode: "CRA", Score: 0.99},
|
||||||
|
{RegulationCode: "CRA", Score: 0.98},
|
||||||
|
{RegulationCode: "CRA", Score: 0.97},
|
||||||
|
{RegulationCode: "NIST", Score: 0.96},
|
||||||
|
{RegulationCode: "MACHINERY", Score: 0.70},
|
||||||
|
{RegulationCode: "MVO", Score: 0.65},
|
||||||
|
}
|
||||||
|
out := balanceByRegulation(pool, regs, 4)
|
||||||
|
var hasCRA, hasMasch bool
|
||||||
|
for _, r := range out {
|
||||||
|
switch r.RegulationCode {
|
||||||
|
case "CRA":
|
||||||
|
hasCRA = true
|
||||||
|
case "MACHINERY", "MVO":
|
||||||
|
hasMasch = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !hasCRA || !hasMasch {
|
||||||
|
t.Errorf("both named domains must be represented: CRA=%v MaschVO=%v out=%v", hasCRA, hasMasch, out)
|
||||||
|
}
|
||||||
|
if out[0].RegulationCode != "CRA" || !(out[1].RegulationCode == "MACHINERY" || out[1].RegulationCode == "MVO") {
|
||||||
|
t.Errorf("round-robin should alternate domains, got %s then %s", out[0].RegulationCode, out[1].RegulationCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDedupResults(t *testing.T) {
|
||||||
|
in := []LegalSearchResult{
|
||||||
|
{RegulationCode: "EDPB WP248", ArticleLabel: "III.B", Text: "lorem", Score: 0.7},
|
||||||
|
{RegulationCode: "EDPB WP248", ArticleLabel: "III.B", Text: "lorem", Score: 0.9}, // dup, higher score
|
||||||
|
{RegulationCode: "DSGVO", ArticleLabel: "Art. 35", Text: "ipsum", Score: 0.8},
|
||||||
|
}
|
||||||
|
out := dedupResults(in)
|
||||||
|
if len(out) != 2 {
|
||||||
|
t.Fatalf("expected 2 deduped, got %d", len(out))
|
||||||
|
}
|
||||||
|
for _, r := range out {
|
||||||
|
if r.RegulationCode == "EDPB WP248" && r.Score != 0.9 {
|
||||||
|
t.Errorf("dedup must keep highest score, got %v", r.Score)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package ucca
|
||||||
|
|
||||||
|
import "context"
|
||||||
|
|
||||||
|
// EvidenceType classifies a retrieved unit by WHAT KIND of evidence it is, independent of its
|
||||||
|
// collection. Footnotes/tables/figures are Evidence Types, not collections. The Authority Router
|
||||||
|
// surfaces non-text evidence from the authoritative knowledge space (the KB slice) SEPARATELY from
|
||||||
|
// the merged text top-K, so fine-grained evidence isn't outranked by broad-base text.
|
||||||
|
//
|
||||||
|
// The layer this introduces: Intent -> Knowledge Space -> EvidenceType -> Collection -> Merge ->
|
||||||
|
// Authority. Today FOOTNOTE is populated; FIGURE arrives with C8 and TABLE is already present from
|
||||||
|
// C6/C9 — no router rebuild needed, the same path carries every new evidence type.
|
||||||
|
type EvidenceType string
|
||||||
|
|
||||||
|
const (
|
||||||
|
EvidenceText EvidenceType = "text"
|
||||||
|
EvidenceFootnote EvidenceType = "footnote"
|
||||||
|
EvidenceTable EvidenceType = "table"
|
||||||
|
EvidenceFigure EvidenceType = "figure"
|
||||||
|
)
|
||||||
|
|
||||||
|
// classifyEvidence derives the EvidenceType from a result's payload markers. Precedence
|
||||||
|
// footnote > figure > table > text (a unit carries at most one is_* marker in practice).
|
||||||
|
func classifyEvidence(r LegalSearchResult) EvidenceType {
|
||||||
|
switch {
|
||||||
|
case r.IsFootnote:
|
||||||
|
return EvidenceFootnote
|
||||||
|
case r.IsFigure:
|
||||||
|
return EvidenceFigure
|
||||||
|
case r.IsTable:
|
||||||
|
return EvidenceTable
|
||||||
|
default:
|
||||||
|
return EvidenceText
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// evidenceRetrievalTopK is the budget for the authoritative-KB evidence pass. Deliberately targeted
|
||||||
|
// (the authoritative slice within the recognized knowledge space), NOT a blanket top-K increase of
|
||||||
|
// the merged result set — the successes came from BETTER-targeted evidence, not MORE evidence.
|
||||||
|
const evidenceRetrievalTopK = 20
|
||||||
|
|
||||||
|
// maxEvidencePerType caps each surfaced evidence type.
|
||||||
|
const maxEvidencePerType = 6
|
||||||
|
|
||||||
|
// RetrieveEvidence returns the authoritative typed evidence (footnotes/tables/figures) for an
|
||||||
|
// in-scope query, pulled from the KB slice and grouped by EvidenceType. This is the "Evidence Type"
|
||||||
|
// router layer (Option A): when the query is in the KB knowledge space, the authoritative evidence
|
||||||
|
// within that space is surfaced separately so it isn't lost in the broad-base text merge. Returns an
|
||||||
|
// empty map when out of scope or KB routing is disabled. Text evidence is NOT returned here — it
|
||||||
|
// flows through the normal Retrieve() merge (the LLM context + the sources list).
|
||||||
|
func (c *LegalRAGClient) RetrieveEvidence(ctx context.Context, query string) map[EvidenceType][]LegalSearchResult {
|
||||||
|
ev := map[EvidenceType][]LegalSearchResult{}
|
||||||
|
if !c.kbScopeRoutingEnabled || c.kbSliceCollection == "" || !inKBScope(query) {
|
||||||
|
return ev
|
||||||
|
}
|
||||||
|
hits, err := c.searchInternal(ctx, c.kbSliceCollection, query, nil, evidenceRetrievalTopK)
|
||||||
|
if err != nil {
|
||||||
|
return ev
|
||||||
|
}
|
||||||
|
for _, h := range hits {
|
||||||
|
t := classifyEvidence(h)
|
||||||
|
if t == EvidenceText || len(ev[t]) >= maxEvidencePerType {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ev[t] = append(ev[t], h)
|
||||||
|
}
|
||||||
|
return ev
|
||||||
|
}
|
||||||
@@ -37,6 +37,17 @@ type LegalSearchResult struct {
|
|||||||
// Supersede-Status (status="superseded", use_for_primary=false) — Alt-Quelle,
|
// Supersede-Status (status="superseded", use_for_primary=false) — Alt-Quelle,
|
||||||
// die fuer Default-Fragen demoted wird (nicht versteckt; fuer Historie auffindbar).
|
// die fuer Default-Fragen demoted wird (nicht versteckt; fuer Historie auffindbar).
|
||||||
Superseded bool `json:"-"`
|
Superseded bool `json:"-"`
|
||||||
|
|
||||||
|
// Evidence-Type-Marker — intern (json:"-", kein Pro-Result-Contract-Change), aus dem
|
||||||
|
// Qdrant-Payload befuellt. classifyEvidence() leitet daraus den EvidenceType ab; der
|
||||||
|
// Router surfacet nicht-Text-Evidence (Fußnote/Tabelle/Figur) getrennt vom Text-Merge,
|
||||||
|
// damit feingranulare Evidence nicht von Breit-Basis-Text ueberrankt wird.
|
||||||
|
IsFootnote bool `json:"-"`
|
||||||
|
FootnoteLabel string `json:"-"`
|
||||||
|
FootnoteVerbatim string `json:"-"`
|
||||||
|
RefCitationUnit string `json:"-"`
|
||||||
|
IsTable bool `json:"-"` // C6/C9: is_table (liniiert + borderless)
|
||||||
|
IsFigure bool `json:"-"` // C8: is_figure (noch nicht befuellt bis C8)
|
||||||
}
|
}
|
||||||
|
|
||||||
// LegalAssessment is the auditable explanation layer over a ranked result set:
|
// LegalAssessment is the auditable explanation layer over a ranked result set:
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ var regulationCatalog = []struct {
|
|||||||
CodeValues []string
|
CodeValues []string
|
||||||
}{
|
}{
|
||||||
{"CRA", []string{"cra", "cyber resilience"}, []string{"CRA"}},
|
{"CRA", []string{"cra", "cyber resilience"}, []string{"CRA"}},
|
||||||
{"MaschVO", []string{"maschinenverordnung", "maschvo", "machinery regulation"}, []string{"MASCHVO", "MaschVO"}},
|
// MaschVO heisst je Collection anders: Slice MASCHVO · gesetze MVO · ce MACHINERY/MASCHINENVO.
|
||||||
|
// Alle Varianten als CodeValues, sonst findet der per-Reg-Filter MaschVO nur in der Slice (0070).
|
||||||
|
{"MaschVO", []string{"maschinenverordnung", "maschvo", "machinery regulation"}, []string{"MASCHVO", "MaschVO", "MVO", "MASCHINENVO", "MACHINERY"}},
|
||||||
{"NIS2", []string{"nis2", "nis-2", "nis 2"}, []string{"NIS2"}},
|
{"NIS2", []string{"nis2", "nis-2", "nis 2"}, []string{"NIS2"}},
|
||||||
{"DORA", []string{"dora"}, []string{"DORA"}},
|
{"DORA", []string{"dora"}, []string{"DORA"}},
|
||||||
{"Data Act", []string{"data act", "datengesetz"}, []string{"DATA ACT", "DataAct"}},
|
{"Data Act", []string{"data act", "datengesetz"}, []string{"DATA ACT", "DataAct"}},
|
||||||
@@ -53,6 +55,62 @@ func detectRegulations(query string) []detectedRegulation {
|
|||||||
|
|
||||||
func hitID(h qdrantSearchHit) string { return fmt.Sprintf("%v", h.ID) }
|
func hitID(h qdrantSearchHit) string { return fmt.Sprintf("%v", h.ID) }
|
||||||
|
|
||||||
|
// balanceByRegulation builds the final top-K so EVERY explicitly-named regulation with hits is
|
||||||
|
// represented, instead of letting the keyword-dominant domain (e.g. CRA) crowd out the other
|
||||||
|
// (e.g. MaschVO) in a cross-regulation query. The input pool must already be score-ordered;
|
||||||
|
// results are grouped by exact regulation_code match against each regulation's CodeValues, then
|
||||||
|
// taken round-robin across the named domains (highest-scored first within each), with any
|
||||||
|
// remaining slots filled by the leftover pool in score order. Generic; no doc-specific logic.
|
||||||
|
func balanceByRegulation(pool []LegalSearchResult, regs []detectedRegulation, topK int) []LegalSearchResult {
|
||||||
|
if topK <= 0 {
|
||||||
|
topK = 8
|
||||||
|
}
|
||||||
|
byReg := make([][]LegalSearchResult, len(regs))
|
||||||
|
matched := make([]bool, len(pool))
|
||||||
|
for ri, r := range regs {
|
||||||
|
for pi := range pool {
|
||||||
|
if matched[pi] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
code := strings.TrimSpace(pool[pi].RegulationCode)
|
||||||
|
for _, cv := range r.CodeValues {
|
||||||
|
if strings.EqualFold(code, cv) {
|
||||||
|
byReg[ri] = append(byReg[ri], pool[pi])
|
||||||
|
matched[pi] = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out := make([]LegalSearchResult, 0, topK)
|
||||||
|
idx := make([]int, len(regs))
|
||||||
|
for len(out) < topK {
|
||||||
|
progressed := false
|
||||||
|
for ri := range regs {
|
||||||
|
if idx[ri] < len(byReg[ri]) {
|
||||||
|
out = append(out, byReg[ri][idx[ri]])
|
||||||
|
idx[ri]++
|
||||||
|
progressed = true
|
||||||
|
if len(out) >= topK {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !progressed {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for pi := range pool {
|
||||||
|
if len(out) >= topK {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !matched[pi] {
|
||||||
|
out = append(out, pool[pi])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// searchMultiRegulation retrieves each explicitly-named regulation SEPARATELY (per-regulation
|
// searchMultiRegulation retrieves each explicitly-named regulation SEPARATELY (per-regulation
|
||||||
// filter) and merges, so a cross-regulation query ("Wie greifen CRA und MaschVO ineinander?")
|
// filter) and merges, so a cross-regulation query ("Wie greifen CRA und MaschVO ineinander?")
|
||||||
// returns BOTH domains in the prompt instead of only the keyword-dominant one. Generic over any
|
// returns BOTH domains in the prompt instead of only the keyword-dominant one. Generic over any
|
||||||
@@ -137,6 +195,13 @@ func hitsToResults(hits []qdrantSearchHit) []LegalSearchResult {
|
|||||||
ReferencesOut: getStringSlice(hit.Payload, "references_out"),
|
ReferencesOut: getStringSlice(hit.Payload, "references_out"),
|
||||||
ReferencesIn: getStringSlice(hit.Payload, "references_in"),
|
ReferencesIn: getStringSlice(hit.Payload, "references_in"),
|
||||||
Superseded: getString(hit.Payload, "status") == "superseded",
|
Superseded: getString(hit.Payload, "status") == "superseded",
|
||||||
|
|
||||||
|
IsFootnote: getBool(hit.Payload, "is_footnote"),
|
||||||
|
FootnoteLabel: getString(hit.Payload, "footnote_label"),
|
||||||
|
FootnoteVerbatim: getString(hit.Payload, "footnote_verbatim"),
|
||||||
|
RefCitationUnit: getString(hit.Payload, "ref_citation_unit"),
|
||||||
|
IsTable: getBool(hit.Payload, "is_table"),
|
||||||
|
IsFigure: getBool(hit.Payload, "is_figure"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return results
|
return results
|
||||||
|
|||||||
Reference in New Issue
Block a user