Files
breakpilot-lehrer/admin-lehrer/app/(admin)/ai/rag/components/ChunkBrowserQA.tsx
Benjamin Admin e7fb9d59f1 Fix ChunkBrowserQA: use regulation_id from Qdrant payload instead of regulation_code
The Qdrant collections use regulation_id (e.g. eu_2016_679) as the filter key,
not regulation_code (e.g. GDPR). Updated rag-constants.ts with correct qdrant_id
mappings from actual Qdrant data, fixed API to filter on regulation_id, and updated
ChunkBrowserQA to pass qdrant_id values.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-28 18:22:12 +01:00

568 lines
23 KiB
TypeScript

'use client'
import React, { useState, useEffect, useCallback, useRef } from 'react'
import { RAG_PDF_MAPPING } from './rag-pdf-mapping'
import { REGULATIONS_IN_RAG, REGULATION_INFO } from '../rag-constants'
interface ChunkBrowserQAProps {
apiProxy: string
}
type RegGroupKey = 'eu_regulation' | 'eu_directive' | 'de_law' | 'at_law' | 'ch_law' | 'national_law' | 'bsi_standard' | 'eu_guideline' | 'international_standard' | 'other'
const GROUP_LABELS: Record<RegGroupKey, string> = {
eu_regulation: 'EU Verordnungen',
eu_directive: 'EU Richtlinien',
de_law: 'DE Gesetze',
at_law: 'AT Gesetze',
ch_law: 'CH Gesetze',
national_law: 'Nationale Gesetze (EU)',
bsi_standard: 'BSI Standards',
eu_guideline: 'EDPB / Guidelines',
international_standard: 'Internationale Standards',
other: 'Sonstige',
}
const GROUP_ORDER: RegGroupKey[] = [
'eu_regulation', 'eu_directive', 'de_law', 'at_law', 'ch_law',
'national_law', 'bsi_standard', 'eu_guideline', 'international_standard', 'other',
]
const COLLECTIONS = [
'bp_compliance_gesetze',
'bp_compliance_ce',
'bp_compliance_datenschutz',
]
export function ChunkBrowserQA({ apiProxy }: ChunkBrowserQAProps) {
// Filter-Sidebar
const [selectedRegulation, setSelectedRegulation] = useState<string | null>(null)
const [regulationCounts, setRegulationCounts] = useState<Record<string, number>>({})
const [filterSearch, setFilterSearch] = useState('')
const [countsLoading, setCountsLoading] = useState(false)
// Dokument-Chunks (sequenziell)
const [docChunks, setDocChunks] = useState<Record<string, unknown>[]>([])
const [docChunkIndex, setDocChunkIndex] = useState(0)
const [docTotalChunks, setDocTotalChunks] = useState(0)
const [docLoading, setDocLoading] = useState(false)
const docChunksRef = useRef(docChunks)
docChunksRef.current = docChunks
// Split-View
const [splitViewActive, setSplitViewActive] = useState(true)
const [chunksPerPage, setChunksPerPage] = useState(6)
// Collection
const [collection, setCollection] = useState('bp_compliance_gesetze')
// Sidebar collapsed groups
const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(new Set())
// Build grouped regulations for sidebar
const regulationsInCollection = Object.entries(REGULATIONS_IN_RAG)
.filter(([, info]) => info.collection === collection)
.map(([code]) => code)
const groupedRegulations = React.useMemo(() => {
const groups: Record<RegGroupKey, { code: string; name: string; type: string }[]> = {
eu_regulation: [], eu_directive: [], de_law: [], at_law: [], ch_law: [],
national_law: [], bsi_standard: [], eu_guideline: [], international_standard: [], other: [],
}
for (const code of regulationsInCollection) {
const reg = REGULATION_INFO.find(r => r.code === code)
const type = (reg?.type || 'other') as RegGroupKey
const groupKey = type in groups ? type : 'other'
groups[groupKey].push({
code,
name: reg?.name || code,
type: reg?.type || 'unknown',
})
}
return groups
}, [regulationsInCollection.join(',')])
// Load regulation counts for current collection
const loadRegulationCounts = useCallback(async (col: string) => {
const entries = Object.entries(REGULATIONS_IN_RAG)
.filter(([, info]) => info.collection === col && info.qdrant_id)
if (entries.length === 0) return
// Build qdrant_id -> our_code mapping
const qdrantIdToCode: Record<string, string[]> = {}
for (const [code, info] of entries) {
if (!qdrantIdToCode[info.qdrant_id]) qdrantIdToCode[info.qdrant_id] = []
qdrantIdToCode[info.qdrant_id].push(code)
}
const uniqueQdrantIds = Object.keys(qdrantIdToCode)
setCountsLoading(true)
try {
const params = new URLSearchParams({
action: 'regulation-counts-batch',
collection: col,
qdrant_ids: uniqueQdrantIds.join(','),
})
const res = await fetch(`${apiProxy}?${params}`)
if (res.ok) {
const data = await res.json()
// Map qdrant_id counts back to our codes
const mapped: Record<string, number> = {}
for (const [qid, count] of Object.entries(data.counts as Record<string, number>)) {
const codes = qdrantIdToCode[qid] || []
for (const code of codes) {
mapped[code] = count
}
}
setRegulationCounts(prev => ({ ...prev, ...mapped }))
}
} catch (error) {
console.error('Failed to load regulation counts:', error)
} finally {
setCountsLoading(false)
}
}, [apiProxy])
// Load all chunks for a regulation (paginated scroll)
const loadDocumentChunks = useCallback(async (regulationCode: string) => {
const ragInfo = REGULATIONS_IN_RAG[regulationCode]
if (!ragInfo || !ragInfo.qdrant_id) return
setDocLoading(true)
setDocChunks([])
setDocChunkIndex(0)
setDocTotalChunks(0)
const allChunks: Record<string, unknown>[] = []
let offset: string | null = null
try {
// Paginated scroll, 100 at a time
let safety = 0
do {
const params = new URLSearchParams({
action: 'scroll',
collection: ragInfo.collection,
limit: '100',
filter_key: 'regulation_id',
filter_value: ragInfo.qdrant_id,
})
if (offset) params.append('offset', offset)
const res = await fetch(`${apiProxy}?${params}`)
if (!res.ok) break
const data = await res.json()
const chunks = data.chunks || []
allChunks.push(...chunks)
offset = data.next_offset || null
safety++
} while (offset && safety < 200) // safety limit ~20k chunks
// Sort by chunk_index
allChunks.sort((a, b) => {
const ai = Number(a.chunk_index ?? a.chunk_id ?? 0)
const bi = Number(b.chunk_index ?? b.chunk_id ?? 0)
return ai - bi
})
setDocChunks(allChunks)
setDocTotalChunks(allChunks.length)
setDocChunkIndex(0)
} catch (error) {
console.error('Failed to load document chunks:', error)
} finally {
setDocLoading(false)
}
}, [apiProxy])
// Initial load
useEffect(() => {
loadRegulationCounts(collection)
}, [collection, loadRegulationCounts])
// Current chunk
const currentChunk = docChunks[docChunkIndex] || null
const prevChunk = docChunkIndex > 0 ? docChunks[docChunkIndex - 1] : null
const nextChunk = docChunkIndex < docChunks.length - 1 ? docChunks[docChunkIndex + 1] : null
// PDF page estimation
const estimatePdfPage = (chunkIndex: number): number => {
const mapping = selectedRegulation ? RAG_PDF_MAPPING[selectedRegulation] : null
const cpp = mapping?.chunksPerPage || chunksPerPage
return Math.floor(chunkIndex / cpp) + 1
}
const pdfPage = currentChunk ? estimatePdfPage(docChunkIndex) : 1
const pdfMapping = selectedRegulation ? RAG_PDF_MAPPING[selectedRegulation] : null
const pdfUrl = pdfMapping ? `/rag-originals/${pdfMapping.filename}#page=${pdfPage}` : null
// Handlers
const handleSelectRegulation = (code: string) => {
setSelectedRegulation(code)
loadDocumentChunks(code)
}
const handleCollectionChange = (col: string) => {
setCollection(col)
setSelectedRegulation(null)
setDocChunks([])
setDocChunkIndex(0)
setDocTotalChunks(0)
setRegulationCounts({})
}
const handlePrev = () => {
if (docChunkIndex > 0) setDocChunkIndex(i => i - 1)
}
const handleNext = () => {
if (docChunkIndex < docChunks.length - 1) setDocChunkIndex(i => i + 1)
}
const handleKeyDown = useCallback((e: KeyboardEvent) => {
if (e.key === 'ArrowLeft' || e.key === 'ArrowUp') {
e.preventDefault()
setDocChunkIndex(i => Math.max(0, i - 1))
} else if (e.key === 'ArrowRight' || e.key === 'ArrowDown') {
e.preventDefault()
setDocChunkIndex(i => Math.min(docChunksRef.current.length - 1, i + 1))
}
}, [])
useEffect(() => {
if (selectedRegulation && docChunks.length > 0) {
window.addEventListener('keydown', handleKeyDown)
return () => window.removeEventListener('keydown', handleKeyDown)
}
}, [selectedRegulation, docChunks.length, handleKeyDown])
const toggleGroup = (group: string) => {
setCollapsedGroups(prev => {
const next = new Set(prev)
if (next.has(group)) next.delete(group)
else next.add(group)
return next
})
}
// Get text content from a chunk
const getChunkText = (chunk: Record<string, unknown> | null): string => {
if (!chunk) return ''
return String(chunk.text || chunk.content || chunk.chunk_text || '')
}
// Overlap extraction
const getOverlapPrev = (): string => {
if (!prevChunk) return ''
const text = getChunkText(prevChunk)
return text.length > 150 ? '...' + text.slice(-150) : text
}
const getOverlapNext = (): string => {
if (!nextChunk) return ''
const text = getChunkText(nextChunk)
return text.length > 150 ? text.slice(0, 150) + '...' : text
}
// Filter sidebar items
const filteredRegulations = React.useMemo(() => {
if (!filterSearch.trim()) return groupedRegulations
const term = filterSearch.toLowerCase()
const filtered: typeof groupedRegulations = {
eu_regulation: [], eu_directive: [], de_law: [], at_law: [], ch_law: [],
national_law: [], bsi_standard: [], eu_guideline: [], international_standard: [], other: [],
}
for (const [group, items] of Object.entries(groupedRegulations)) {
filtered[group as RegGroupKey] = items.filter(
r => r.code.toLowerCase().includes(term) || r.name.toLowerCase().includes(term)
)
}
return filtered
}, [groupedRegulations, filterSearch])
// Regulation name lookup
const getRegName = (code: string): string => {
const reg = REGULATION_INFO.find(r => r.code === code)
return reg?.name || code
}
return (
<div className="space-y-4">
{/* Header bar */}
<div className="bg-white rounded-xl border border-slate-200 p-4">
<div className="flex flex-wrap items-center gap-4">
<div>
<label className="block text-xs font-medium text-slate-500 mb-1">Collection</label>
<select
value={collection}
onChange={(e) => handleCollectionChange(e.target.value)}
className="px-3 py-1.5 border rounded-lg text-sm focus:ring-2 focus:ring-teal-500"
>
{COLLECTIONS.map(c => (
<option key={c} value={c}>{c}</option>
))}
</select>
</div>
{selectedRegulation && (
<>
<div className="flex items-center gap-2">
<span className="text-sm font-semibold text-slate-900">
QA-Modus: {selectedRegulation} {getRegName(selectedRegulation)}
</span>
</div>
<div className="flex items-center gap-2 ml-auto">
<span className="text-sm text-slate-600">
Chunk {docChunkIndex + 1} / {docTotalChunks}
</span>
<button
onClick={handlePrev}
disabled={docChunkIndex === 0}
className="px-3 py-1 text-sm border rounded-lg bg-white hover:bg-slate-50 disabled:opacity-30"
>
&#9664; Prev
</button>
<button
onClick={handleNext}
disabled={docChunkIndex >= docChunks.length - 1}
className="px-3 py-1 text-sm border rounded-lg bg-white hover:bg-slate-50 disabled:opacity-30"
>
Next &#9654;
</button>
</div>
<div className="flex items-center gap-2">
<label className="text-xs text-slate-500">Chunks/Seite:</label>
<select
value={chunksPerPage}
onChange={(e) => setChunksPerPage(Number(e.target.value))}
className="px-2 py-1 border rounded text-xs"
>
{[3, 4, 5, 6, 8, 10, 12, 15, 20].map(n => (
<option key={n} value={n}>{n}</option>
))}
</select>
<button
onClick={() => setSplitViewActive(!splitViewActive)}
className={`px-3 py-1 text-xs rounded-lg border ${
splitViewActive ? 'bg-teal-50 border-teal-300 text-teal-700' : 'bg-slate-50 border-slate-300 text-slate-600'
}`}
>
{splitViewActive ? 'Split-View an' : 'Split-View aus'}
</button>
</div>
</>
)}
</div>
</div>
{/* Main content: Sidebar + Content */}
<div className="flex gap-4" style={{ minHeight: '70vh' }}>
{/* Sidebar */}
<div className="w-64 flex-shrink-0 bg-white rounded-xl border border-slate-200 overflow-hidden flex flex-col">
<div className="p-3 border-b border-slate-100">
<input
type="text"
value={filterSearch}
onChange={(e) => setFilterSearch(e.target.value)}
placeholder="Suche..."
className="w-full px-2 py-1.5 border rounded-lg text-sm focus:ring-2 focus:ring-teal-500"
/>
{countsLoading && (
<div className="text-xs text-slate-400 mt-1">Counts werden geladen...</div>
)}
</div>
<div className="flex-1 overflow-y-auto">
{GROUP_ORDER.map(group => {
const items = filteredRegulations[group]
if (items.length === 0) return null
const isCollapsed = collapsedGroups.has(group)
return (
<div key={group}>
<button
onClick={() => toggleGroup(group)}
className="w-full px-3 py-1.5 text-left text-xs font-semibold text-slate-500 bg-slate-50 hover:bg-slate-100 flex items-center justify-between"
>
<span>{GROUP_LABELS[group]}</span>
<span className="text-slate-400">{isCollapsed ? '+' : '-'}</span>
</button>
{!isCollapsed && items.map(reg => {
const count = regulationCounts[reg.code] ?? REGULATIONS_IN_RAG[reg.code]?.chunks ?? 0
const isSelected = selectedRegulation === reg.code
return (
<button
key={reg.code}
onClick={() => handleSelectRegulation(reg.code)}
className={`w-full px-3 py-1.5 text-left text-sm flex items-center justify-between hover:bg-teal-50 transition-colors ${
isSelected ? 'bg-teal-100 text-teal-900 font-medium' : 'text-slate-700'
}`}
>
<span className="truncate">{reg.code}</span>
<span className={`text-xs tabular-nums ${count > 0 ? 'text-slate-500' : 'text-slate-300'}`}>
{count > 0 ? count.toLocaleString() : '—'}
</span>
</button>
)
})}
</div>
)
})}
</div>
</div>
{/* Content area */}
{!selectedRegulation ? (
<div className="flex-1 flex items-center justify-center bg-white rounded-xl border border-slate-200">
<div className="text-center text-slate-400 space-y-2">
<div className="text-4xl">&#128269;</div>
<p className="text-sm">Waehle ein Dokument in der Sidebar, um die QA-Ansicht zu starten.</p>
<p className="text-xs text-slate-300">Pfeiltasten navigieren zwischen Chunks.</p>
</div>
</div>
) : docLoading ? (
<div className="flex-1 flex items-center justify-center bg-white rounded-xl border border-slate-200">
<div className="text-center text-slate-500 space-y-2">
<div className="animate-spin text-3xl">&#9881;</div>
<p className="text-sm">Chunks werden geladen...</p>
<p className="text-xs text-slate-400">
{selectedRegulation}: {REGULATIONS_IN_RAG[selectedRegulation]?.chunks || '?'} Chunks erwartet
</p>
</div>
</div>
) : (
<div className={`flex-1 grid gap-4 ${splitViewActive ? 'grid-cols-2' : 'grid-cols-1'}`}>
{/* Chunk-Text Panel */}
<div className="bg-white rounded-xl border border-slate-200 overflow-hidden flex flex-col">
<div className="px-4 py-2 bg-slate-50 border-b border-slate-100 flex items-center justify-between">
<span className="text-sm font-medium text-slate-700">Chunk-Text</span>
<span className="text-xs text-slate-400">
Index: {docChunkIndex} / {docTotalChunks - 1}
</span>
</div>
<div className="flex-1 overflow-y-auto p-4 space-y-3">
{/* Overlap from previous chunk */}
{prevChunk && (
<div className="text-xs text-slate-400 bg-amber-50 border-l-2 border-amber-300 px-3 py-2 rounded-r">
<div className="font-medium text-amber-600 mb-1">&#8593; Overlap (vorheriger Chunk #{docChunkIndex - 1})</div>
<p className="whitespace-pre-wrap break-words leading-relaxed">{getOverlapPrev()}</p>
</div>
)}
{/* Current chunk text */}
{currentChunk && (
<div className="text-sm text-slate-800 whitespace-pre-wrap break-words leading-relaxed border-l-2 border-teal-400 pl-3">
{getChunkText(currentChunk)}
</div>
)}
{/* Overlap from next chunk */}
{nextChunk && (
<div className="text-xs text-slate-400 bg-amber-50 border-l-2 border-amber-300 px-3 py-2 rounded-r">
<div className="font-medium text-amber-600 mb-1">&#8595; Overlap (naechster Chunk #{docChunkIndex + 1})</div>
<p className="whitespace-pre-wrap break-words leading-relaxed">{getOverlapNext()}</p>
</div>
)}
{/* Metadata */}
{currentChunk && (
<div className="mt-4 pt-3 border-t border-slate-100">
<div className="text-xs font-medium text-slate-500 mb-2">Metadaten</div>
<div className="grid grid-cols-2 gap-x-4 gap-y-1 text-xs">
{Object.entries(currentChunk)
.filter(([k]) => !['text', 'content', 'chunk_text', 'id'].includes(k))
.map(([k, v]) => (
<div key={k} className="flex gap-1">
<span className="font-medium text-slate-500">{k}:</span>
<span className="text-slate-700 truncate">{String(v)}</span>
</div>
))}
</div>
</div>
)}
</div>
{/* Bottom nav */}
<div className="px-4 py-2 border-t border-slate-100 bg-slate-50 flex items-center justify-between">
<button
onClick={handlePrev}
disabled={docChunkIndex === 0}
className="px-3 py-1 text-xs border rounded bg-white hover:bg-slate-50 disabled:opacity-30"
>
&#9664; Zurueck
</button>
<div className="flex items-center gap-3">
<input
type="number"
min={0}
max={docTotalChunks - 1}
value={docChunkIndex}
onChange={(e) => {
const v = parseInt(e.target.value, 10)
if (!isNaN(v) && v >= 0 && v < docTotalChunks) setDocChunkIndex(v)
}}
className="w-20 px-2 py-1 border rounded text-xs text-center"
/>
<span className="text-xs text-slate-400">/ {docTotalChunks - 1}</span>
</div>
<button
onClick={handleNext}
disabled={docChunkIndex >= docChunks.length - 1}
className="px-3 py-1 text-xs border rounded bg-white hover:bg-slate-50 disabled:opacity-30"
>
Weiter &#9654;
</button>
</div>
</div>
{/* PDF-Viewer Panel */}
{splitViewActive && (
<div className="bg-white rounded-xl border border-slate-200 overflow-hidden flex flex-col">
<div className="px-4 py-2 bg-slate-50 border-b border-slate-100 flex items-center justify-between">
<span className="text-sm font-medium text-slate-700">Original-PDF</span>
<span className="text-xs text-slate-400">
Seite ~{pdfPage}
{pdfMapping?.totalPages ? ` / ${pdfMapping.totalPages}` : ''}
</span>
</div>
<div className="flex-1 relative">
{pdfUrl ? (
<iframe
key={`${selectedRegulation}-${pdfPage}`}
src={pdfUrl}
className="absolute inset-0 w-full h-full border-0"
title="Original PDF"
/>
) : (
<div className="flex items-center justify-center h-full text-slate-400 text-sm">
<div className="text-center space-y-2">
<div className="text-3xl">&#128196;</div>
<p>Kein PDF-Mapping fuer {selectedRegulation}.</p>
<p className="text-xs">Bitte rag-pdf-mapping.ts ergaenzen und PDF in ~/rag-originals/ ablegen.</p>
</div>
</div>
)}
</div>
{pdfUrl && (
<div className="px-4 py-2 border-t border-slate-100 bg-slate-50 flex items-center justify-between">
<span className="text-xs text-slate-500">
{pdfMapping?.filename}
</span>
<a
href={pdfUrl.split('#')[0]}
target="_blank"
rel="noopener noreferrer"
className="text-xs text-teal-600 hover:text-teal-800"
>
PDF oeffnen &#8599;
</a>
</div>
)}
</div>
)}
</div>
)}
</div>
</div>
)
}