feat: add Chunk-Browser tab to RAG page

- New 'Chunk-Browser' tab for sequential chunk browsing
- Qdrant scroll API proxy (scroll + collection-count actions)
- Pagination with prev/next through all chunks in a collection
- Text search filter with highlighting
- Click to expand chunk and see all metadata
- 'In Chunks suchen' button now navigates to Chunk-Browser with correct collection

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-02-28 09:35:52 +01:00
parent 954103cdf2
commit 491df4e1b0
2 changed files with 317 additions and 3 deletions

View File

@@ -73,7 +73,7 @@ interface DsfaCorpusStatus {
type RegulationCategory = 'regulations' | 'dsfa' | 'nibis' | 'templates'
// Tab definitions
type TabId = 'overview' | 'regulations' | 'map' | 'search' | 'data' | 'ingestion' | 'pipeline'
type TabId = 'overview' | 'regulations' | 'map' | 'search' | 'chunks' | 'data' | 'ingestion' | 'pipeline'
// Custom document type
interface CustomDocument {
@@ -1850,6 +1850,18 @@ export default function RAGPage() {
const [autoRefresh, setAutoRefresh] = useState(true)
const [elapsedTime, setElapsedTime] = useState<string>('')
// Chunk browser state
const [chunkCollection, setChunkCollection] = useState('bp_compliance_gesetze')
const [chunkData, setChunkData] = useState<Record<string, unknown>[]>([])
const [chunkOffset, setChunkOffset] = useState<string | null>(null)
const [chunkHistory, setChunkHistory] = useState<(string | null)[]>([])
const [chunkLoading, setChunkLoading] = useState(false)
const [chunkTextSearch, setChunkTextSearch] = useState('')
const [chunkTotalCount, setChunkTotalCount] = useState(0)
const [chunkCurrentPage, setChunkCurrentPage] = useState(0)
const [chunkNextOffset, setChunkNextOffset] = useState<string | null>(null)
const [expandedChunk, setExpandedChunk] = useState<number | null>(null)
// DSFA corpus state
const [dsfaSources, setDsfaSources] = useState<DsfaSource[]>([])
const [dsfaStatus, setDsfaStatus] = useState<DsfaCorpusStatus | null>(null)
@@ -2095,6 +2107,69 @@ export default function RAGPage() {
return () => clearInterval(interval)
}, [pipelineState?.started_at, pipelineState?.status])
const loadChunks = async (offset: string | null = null, newCollection?: string) => {
const col = newCollection || chunkCollection
setChunkLoading(true)
try {
const params = new URLSearchParams({
action: 'scroll',
collection: col,
limit: '20',
})
if (offset) params.append('offset', offset)
if (chunkTextSearch.trim()) params.append('text_search', chunkTextSearch.trim())
const res = await fetch(`${API_PROXY}?${params}`)
if (res.ok) {
const data = await res.json()
setChunkData(data.chunks || [])
setChunkNextOffset(data.next_offset || null)
setExpandedChunk(null)
}
} catch (error) {
console.error('Chunk scroll failed:', error)
} finally {
setChunkLoading(false)
}
}
const loadChunkCount = async (col: string) => {
try {
const res = await fetch(`${API_PROXY}?action=collection-count&collection=${encodeURIComponent(col)}`)
if (res.ok) {
const data = await res.json()
setChunkTotalCount(data.count || 0)
}
} catch { /* ignore */ }
}
const handleChunkCollectionChange = (col: string) => {
setChunkCollection(col)
setChunkOffset(null)
setChunkHistory([])
setChunkCurrentPage(0)
loadChunkCount(col)
loadChunks(null, col)
}
const handleChunkNext = () => {
if (!chunkNextOffset) return
setChunkHistory((prev) => [...prev, chunkOffset])
setChunkOffset(chunkNextOffset)
setChunkCurrentPage((p) => p + 1)
loadChunks(chunkNextOffset)
}
const handleChunkPrev = () => {
if (chunkHistory.length === 0) return
const prev = [...chunkHistory]
const prevOffset = prev.pop() ?? null
setChunkHistory(prev)
setChunkOffset(prevOffset)
setChunkCurrentPage((p) => Math.max(0, p - 1))
loadChunks(prevOffset)
}
const handleSearch = async () => {
if (!searchQuery.trim()) return
@@ -2180,6 +2255,7 @@ export default function RAGPage() {
{ id: 'regulations' as TabId, name: 'Regulierungen', icon: '📜' },
{ id: 'map' as TabId, name: 'Landkarte', icon: '🗺️' },
{ id: 'search' as TabId, name: 'Suche', icon: '🔍' },
{ id: 'chunks' as TabId, name: 'Chunk-Browser', icon: '🧩' },
{ id: 'data' as TabId, name: 'Daten', icon: '📁' },
{ id: 'ingestion' as TabId, name: 'Ingestion', icon: '⚙️' },
{ id: 'pipeline' as TabId, name: 'Pipeline', icon: '🔄' },
@@ -2535,8 +2611,11 @@ export default function RAGPage() {
<button
onClick={(e) => {
e.stopPropagation()
setSearchQuery(reg.name)
setActiveTab('search')
const ragEntry = REGULATIONS_IN_RAG[reg.code as keyof typeof REGULATIONS_IN_RAG]
const col = ragEntry?.collection || 'bp_compliance_gesetze'
setChunkTextSearch(reg.name)
handleChunkCollectionChange(col)
setActiveTab('chunks')
}}
className="text-teal-600 hover:text-teal-700 font-medium"
>
@@ -3183,6 +3262,175 @@ export default function RAGPage() {
</div>
)}
{activeTab === 'chunks' && (
<div className="space-y-6">
{/* Collection Selector + Controls */}
<div className="bg-white rounded-xl border border-slate-200 p-6">
<h3 className="font-semibold text-slate-900 mb-4">Chunk-Browser</h3>
<div className="flex flex-wrap gap-4 items-end">
<div>
<label className="block text-sm font-medium text-slate-700 mb-1">Collection</label>
<select
value={chunkCollection}
onChange={(e) => handleChunkCollectionChange(e.target.value)}
className="px-3 py-2 border rounded-lg text-sm focus:ring-2 focus:ring-teal-500"
>
<option value="bp_compliance_gesetze">bp_compliance_gesetze</option>
<option value="bp_compliance_ce">bp_compliance_ce</option>
<option value="bp_compliance_datenschutz">bp_compliance_datenschutz</option>
<option value="bp_dsfa_corpus">bp_dsfa_corpus</option>
<option value="bp_compliance_recht">bp_compliance_recht</option>
<option value="bp_legal_templates">bp_legal_templates</option>
<option value="bp_compliance_gdpr">bp_compliance_gdpr</option>
<option value="bp_compliance_schulrecht">bp_compliance_schulrecht</option>
<option value="bp_dsfa_templates">bp_dsfa_templates</option>
<option value="bp_dsfa_risks">bp_dsfa_risks</option>
</select>
</div>
<div className="flex-1 min-w-[200px]">
<label className="block text-sm font-medium text-slate-700 mb-1">Textsuche (filtert geladene Seite)</label>
<div className="flex gap-2">
<input
type="text"
value={chunkTextSearch}
onChange={(e) => setChunkTextSearch(e.target.value)}
onKeyDown={(e) => { if (e.key === 'Enter') loadChunks(null) }}
placeholder="z.B. DSGVO, IFRS, Maschinenverordnung..."
className="flex-1 px-3 py-2 border rounded-lg text-sm focus:ring-2 focus:ring-teal-500"
/>
<button
onClick={() => { setChunkOffset(null); setChunkHistory([]); setChunkCurrentPage(0); loadChunks(null) }}
className="px-4 py-2 bg-teal-600 text-white text-sm rounded-lg hover:bg-teal-700"
>
Laden
</button>
</div>
</div>
<div className="text-sm text-slate-500">
{chunkTotalCount > 0 && <span>{chunkTotalCount.toLocaleString()} Chunks total</span>}
</div>
</div>
</div>
{/* Pagination */}
{chunkData.length > 0 && (
<div className="flex items-center justify-between">
<button
onClick={handleChunkPrev}
disabled={chunkCurrentPage === 0}
className="px-4 py-2 text-sm border rounded-lg bg-white hover:bg-slate-50 disabled:opacity-30"
>
Zurueck
</button>
<span className="text-sm text-slate-600">
Seite {chunkCurrentPage + 1} {chunkData.length} Chunks angezeigt
</span>
<button
onClick={handleChunkNext}
disabled={!chunkNextOffset}
className="px-4 py-2 text-sm border rounded-lg bg-white hover:bg-slate-50 disabled:opacity-30"
>
Weiter
</button>
</div>
)}
{/* Chunk List */}
{chunkLoading ? (
<div className="text-center py-12 text-slate-500">Chunks werden geladen...</div>
) : chunkData.length === 0 ? (
<div className="text-center py-12 text-slate-400">
Collection waehlen und &quot;Laden&quot; klicken um Chunks anzuzeigen.
</div>
) : (
<div className="space-y-2">
{chunkData.map((chunk, i) => {
const text = String(chunk.text || chunk.content || chunk.chunk_text || '')
const source = String(chunk.source_name || chunk.regulation_code || chunk.guideline_name || chunk.regulation_short || '')
const isExpanded = expandedChunk === i
const highlightTerm = chunkTextSearch.trim().toLowerCase()
const renderHighlighted = (str: string) => {
if (!highlightTerm) return str
const idx = str.toLowerCase().indexOf(highlightTerm)
if (idx === -1) return str
return (
<>
{str.slice(0, idx)}
<mark className="bg-yellow-200 px-0.5 rounded">{str.slice(idx, idx + highlightTerm.length)}</mark>
{str.slice(idx + highlightTerm.length)}
</>
)
}
return (
<div
key={String(chunk.id || i)}
className={`bg-white rounded-lg border transition-all cursor-pointer ${
isExpanded ? 'border-teal-300 shadow-md' : 'border-slate-200 hover:border-slate-300'
}`}
onClick={() => setExpandedChunk(isExpanded ? null : i)}
>
<div className="px-4 py-3">
<div className="flex items-center gap-2 mb-1">
<span className="text-xs font-mono text-slate-400">#{chunkCurrentPage * 20 + i + 1}</span>
{source && (
<span className="px-2 py-0.5 text-xs rounded bg-slate-100 text-slate-600">{source}</span>
)}
{chunk.article && (
<span className="text-xs text-slate-500">Art. {String(chunk.article)}</span>
)}
{chunk.language && (
<span className="text-xs text-slate-400 ml-auto">{String(chunk.language).toUpperCase()}</span>
)}
</div>
<p className={`text-sm text-slate-700 ${isExpanded ? '' : 'line-clamp-3'}`}>
{renderHighlighted(text)}
</p>
</div>
{isExpanded && (
<div className="px-4 py-3 border-t border-slate-100 bg-slate-50 text-xs text-slate-500 space-y-1">
<div className="grid grid-cols-2 md:grid-cols-4 gap-2">
{Object.entries(chunk).filter(([k]) => !['text', 'content', 'chunk_text', 'id'].includes(k)).map(([k, v]) => (
<div key={k}>
<span className="font-medium text-slate-600">{k}:</span>{' '}
<span>{String(v)}</span>
</div>
))}
</div>
</div>
)}
</div>
)
})}
</div>
)}
{/* Bottom Pagination */}
{chunkData.length > 0 && (
<div className="flex items-center justify-between">
<button
onClick={handleChunkPrev}
disabled={chunkCurrentPage === 0}
className="px-4 py-2 text-sm border rounded-lg bg-white hover:bg-slate-50 disabled:opacity-30"
>
Zurueck
</button>
<span className="text-sm text-slate-600">
Seite {chunkCurrentPage + 1}
</span>
<button
onClick={handleChunkNext}
disabled={!chunkNextOffset}
className="px-4 py-2 text-sm border rounded-lg bg-white hover:bg-slate-50 disabled:opacity-30"
>
Weiter
</button>
</div>
)}
</div>
)}
{activeTab === 'data' && (
<div className="space-y-6">
{/* Upload Document */}

View File

@@ -66,6 +66,72 @@ export async function GET(request: NextRequest) {
url += `/traceability?chunk_id=${encodeURIComponent(chunkId || '')}&regulation=${encodeURIComponent(regulation || '')}`
break
}
case 'scroll': {
const collection = searchParams.get('collection') || 'bp_compliance_gesetze'
const limit = parseInt(searchParams.get('limit') || '20', 10)
const offsetParam = searchParams.get('offset')
const filterKey = searchParams.get('filter_key')
const filterValue = searchParams.get('filter_value')
const textSearch = searchParams.get('text_search')
const scrollBody: Record<string, unknown> = {
limit: Math.min(limit, 100),
with_payload: true,
with_vector: false,
}
if (offsetParam) {
scrollBody.offset = offsetParam
}
if (filterKey && filterValue) {
scrollBody.filter = {
must: [{ key: filterKey, match: { value: filterValue } }],
}
}
const scrollRes = await fetch(`${QDRANT_URL}/collections/${encodeURIComponent(collection)}/points/scroll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(scrollBody),
cache: 'no-store',
})
if (!scrollRes.ok) {
return NextResponse.json({ error: 'Qdrant scroll failed' }, { status: scrollRes.status })
}
const scrollData = await scrollRes.json()
const points = (scrollData.result?.points || []).map((p: { id: string; payload?: Record<string, unknown> }) => ({
id: p.id,
...p.payload,
}))
// Client-side text search filter
let filtered = points
if (textSearch && textSearch.trim()) {
const term = textSearch.toLowerCase()
filtered = points.filter((p: Record<string, unknown>) => {
const text = String(p.text || p.content || p.chunk_text || '')
return text.toLowerCase().includes(term)
})
}
return NextResponse.json({
chunks: filtered,
next_offset: scrollData.result?.next_page_offset || null,
total_in_page: points.length,
})
}
case 'collection-count': {
const col = searchParams.get('collection') || 'bp_compliance_gesetze'
const countRes = await fetch(`${QDRANT_URL}/collections/${encodeURIComponent(col)}`, {
cache: 'no-store',
})
if (!countRes.ok) {
return NextResponse.json({ count: 0 })
}
const countData = await countRes.json()
return NextResponse.json({
count: countData.result?.points_count || 0,
})
}
default:
return NextResponse.json({ error: 'Unknown action' }, { status: 400 })
}