feat(control-library): document-grouped batching, generation strategy tracking, sort by source

- Group chunks by regulation_code before batching for better LLM context
- Add generation_strategy column (ungrouped=v1, document_grouped=v2)
- Add v1/v2 badge to control cards in frontend
- Add sort-by-source option with visual group headers
- Add frontend page tests (18 tests)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

admin-compliance/app/sdk/control-library/page.tsx

@@ -1,33 +1,50 @@
 'use client'
 
-import { useState, useEffect, useMemo, useCallback } from 'react'
+import { useState, useEffect, useCallback, useRef } from 'react'
 import {
   Shield, Search, ChevronRight, ChevronLeft, Filter, Lock,
   BookOpen, Plus, Zap, BarChart3, ListChecks,
-  ChevronsLeft, ChevronsRight,
+  ChevronsLeft, ChevronsRight, ArrowUpDown, Clock, RefreshCw,
 } from 'lucide-react'
 import {
   CanonicalControl, Framework, BACKEND_URL, EMPTY_CONTROL,
   SeverityBadge, StateBadge, LicenseRuleBadge, VerificationMethodBadge, CategoryBadge, TargetAudienceBadge,
-  getDomain, VERIFICATION_METHODS, CATEGORY_OPTIONS, TARGET_AUDIENCE_OPTIONS,
+  GenerationStrategyBadge,
+  VERIFICATION_METHODS, CATEGORY_OPTIONS, TARGET_AUDIENCE_OPTIONS,
 } from './components/helpers'
 import { ControlForm } from './components/ControlForm'
 import { ControlDetail } from './components/ControlDetail'
 import { GeneratorModal } from './components/GeneratorModal'
 
 // =============================================================================
-// CONTROL LIBRARY PAGE
+// TYPES
 // =============================================================================
 
+interface ControlsMeta {
+  total: number
+  domains: Array<{ domain: string; count: number }>
+  sources: Array<{ source: string; count: number }>
+  no_source_count: number
+}
+
+// =============================================================================
+// CONTROL LIBRARY PAGE — Server-Side Pagination
+// =============================================================================
+
+const PAGE_SIZE = 50
+
 export default function ControlLibraryPage() {
   const [frameworks, setFrameworks] = useState<Framework[]>([])
   const [controls, setControls] = useState<CanonicalControl[]>([])
+  const [totalCount, setTotalCount] = useState(0)
+  const [meta, setMeta] = useState<ControlsMeta | null>(null)
   const [selectedControl, setSelectedControl] = useState<CanonicalControl | null>(null)
   const [loading, setLoading] = useState(true)
   const [error, setError] = useState<string | null>(null)
 
   // Filters
   const [searchQuery, setSearchQuery] = useState('')
+  const [debouncedSearch, setDebouncedSearch] = useState('')
   const [severityFilter, setSeverityFilter] = useState<string>('')
   const [domainFilter, setDomainFilter] = useState<string>('')
   const [stateFilter, setStateFilter] = useState<string>('')
@@ -35,6 +52,7 @@ export default function ControlLibraryPage() {
   const [categoryFilter, setCategoryFilter] = useState<string>('')
   const [audienceFilter, setAudienceFilter] = useState<string>('')
   const [sourceFilter, setSourceFilter] = useState<string>('')
+  const [sortBy, setSortBy] = useState<'id' | 'newest' | 'oldest' | 'source'>('id')
 
   // CRUD state
   const [mode, setMode] = useState<'list' | 'detail' | 'create' | 'edit'>('list')
@@ -47,98 +65,111 @@ export default function ControlLibraryPage() {
 
   // Pagination
   const [currentPage, setCurrentPage] = useState(1)
-  const PAGE_SIZE = 50
 
   // Review mode
   const [reviewMode, setReviewMode] = useState(false)
   const [reviewIndex, setReviewIndex] = useState(0)
+  const [reviewItems, setReviewItems] = useState<CanonicalControl[]>([])
+  const [reviewCount, setReviewCount] = useState(0)
 
-  // Load data
-  const loadData = useCallback(async () => {
+  // Debounce search
+  const searchTimer = useRef<ReturnType<typeof setTimeout> | null>(null)
+  useEffect(() => {
+    if (searchTimer.current) clearTimeout(searchTimer.current)
+    searchTimer.current = setTimeout(() => setDebouncedSearch(searchQuery), 400)
+    return () => { if (searchTimer.current) clearTimeout(searchTimer.current) }
+  }, [searchQuery])
+
+  // Build query params for backend
+  const buildParams = useCallback((extra?: Record<string, string>) => {
+    const p = new URLSearchParams()
+    if (severityFilter) p.set('severity', severityFilter)
+    if (domainFilter) p.set('domain', domainFilter)
+    if (stateFilter) p.set('release_state', stateFilter)
+    if (verificationFilter) p.set('verification_method', verificationFilter)
+    if (categoryFilter) p.set('category', categoryFilter)
+    if (audienceFilter) p.set('target_audience', audienceFilter)
+    if (sourceFilter) p.set('source', sourceFilter)
+    if (debouncedSearch) p.set('search', debouncedSearch)
+    if (extra) for (const [k, v] of Object.entries(extra)) p.set(k, v)
+    return p.toString()
+  }, [severityFilter, domainFilter, stateFilter, verificationFilter, categoryFilter, audienceFilter, sourceFilter, debouncedSearch])
+
+  // Load metadata (domains, sources — once + on refresh)
+  const loadMeta = useCallback(async () => {
+    try {
+      const [fwRes, metaRes] = await Promise.all([
+        fetch(`${BACKEND_URL}?endpoint=frameworks`),
+        fetch(`${BACKEND_URL}?endpoint=controls-meta`),
+      ])
+      if (fwRes.ok) setFrameworks(await fwRes.json())
+      if (metaRes.ok) setMeta(await metaRes.json())
+    } catch { /* ignore */ }
+  }, [])
+
+  // Load controls page
+  const loadControls = useCallback(async () => {
     try {
       setLoading(true)
-      const [fwRes, ctrlRes] = await Promise.all([
-        fetch(`${BACKEND_URL}?endpoint=frameworks`),
-        fetch(`${BACKEND_URL}?endpoint=controls`),
+
+      // Determine sort
+      const sortField = sortBy === 'id' ? 'control_id' : sortBy === 'source' ? 'source' : 'created_at'
+      const sortOrder = sortBy === 'newest' ? 'desc' : sortBy === 'oldest' ? 'asc' : 'asc'
+      const offset = (currentPage - 1) * PAGE_SIZE
+
+      const qs = buildParams({
+        sort: sortField,
+        order: sortOrder,
+        limit: String(PAGE_SIZE),
+        offset: String(offset),
+      })
+
+      const countQs = buildParams()
+
+      const [ctrlRes, countRes] = await Promise.all([
+        fetch(`${BACKEND_URL}?endpoint=controls&${qs}`),
+        fetch(`${BACKEND_URL}?endpoint=controls-count&${countQs}`),
       ])
 
-      if (fwRes.ok) setFrameworks(await fwRes.json())
       if (ctrlRes.ok) setControls(await ctrlRes.json())
+      if (countRes.ok) {
+        const data = await countRes.json()
+        setTotalCount(data.total || 0)
+      }
     } catch (err) {
       setError(err instanceof Error ? err.message : 'Fehler beim Laden')
     } finally {
       setLoading(false)
     }
+  }, [buildParams, sortBy, currentPage])
+
+  // Load review count
+  const loadReviewCount = useCallback(async () => {
+    try {
+      const res = await fetch(`${BACKEND_URL}?endpoint=controls-count&release_state=needs_review`)
+      if (res.ok) {
+        const data = await res.json()
+        setReviewCount(data.total || 0)
+      }
+    } catch { /* ignore */ }
   }, [])
 
-  useEffect(() => { loadData() }, [loadData])
+  // Initial load
+  useEffect(() => { loadMeta(); loadReviewCount() }, [loadMeta, loadReviewCount])
 
-  // Derived: unique domains
-  const domains = useMemo(() => {
-    const set = new Set(controls.map(c => getDomain(c.control_id)))
-    return Array.from(set).sort()
-  }, [controls])
-
-  // Derived: unique document sources (sorted by frequency)
-  const documentSources = useMemo(() => {
-    const counts = new Map<string, number>()
-    let noSource = 0
-    for (const c of controls) {
-      const src = c.source_citation?.source
-      if (src) {
-        counts.set(src, (counts.get(src) || 0) + 1)
-      } else {
-        noSource++
-      }
-    }
-    const sorted = Array.from(counts.entries()).sort((a, b) => b[1] - a[1])
-    return { sources: sorted, noSourceCount: noSource }
-  }, [controls])
-
-  // Filtered controls
-  const filteredControls = useMemo(() => {
-    return controls.filter(c => {
-      if (severityFilter && c.severity !== severityFilter) return false
-      if (domainFilter && getDomain(c.control_id) !== domainFilter) return false
-      if (stateFilter && c.release_state !== stateFilter) return false
-      if (verificationFilter && c.verification_method !== verificationFilter) return false
-      if (categoryFilter && c.category !== categoryFilter) return false
-      if (audienceFilter && c.target_audience !== audienceFilter) return false
-      if (sourceFilter) {
-        const src = c.source_citation?.source || ''
-        if (sourceFilter === '__none__') {
-          if (src) return false
-        } else {
-          if (src !== sourceFilter) return false
-        }
-      }
-      if (searchQuery) {
-        const q = searchQuery.toLowerCase()
-        return (
-          c.control_id.toLowerCase().includes(q) ||
-          c.title.toLowerCase().includes(q) ||
-          c.objective.toLowerCase().includes(q) ||
-          c.tags.some(t => t.toLowerCase().includes(q))
-        )
-      }
-      return true
-    })
-  }, [controls, severityFilter, domainFilter, stateFilter, verificationFilter, categoryFilter, audienceFilter, sourceFilter, searchQuery])
+  // Load controls when filters/page/sort change
+  useEffect(() => { loadControls() }, [loadControls])
 
   // Reset page when filters change
-  useEffect(() => { setCurrentPage(1) }, [severityFilter, domainFilter, stateFilter, verificationFilter, categoryFilter, audienceFilter, sourceFilter, searchQuery])
+  useEffect(() => { setCurrentPage(1) }, [severityFilter, domainFilter, stateFilter, verificationFilter, categoryFilter, audienceFilter, sourceFilter, debouncedSearch, sortBy])
 
   // Pagination
-  const totalPages = Math.max(1, Math.ceil(filteredControls.length / PAGE_SIZE))
-  const paginatedControls = useMemo(() => {
-    const start = (currentPage - 1) * PAGE_SIZE
-    return filteredControls.slice(start, start + PAGE_SIZE)
-  }, [filteredControls, currentPage])
+  const totalPages = Math.max(1, Math.ceil(totalCount / PAGE_SIZE))
 
-  // Review queue items
-  const reviewItems = useMemo(() => {
-    return controls.filter(c => ['needs_review', 'too_close', 'duplicate'].includes(c.release_state))
-  }, [controls])
+  // Full reload (after CRUD)
+  const fullReload = useCallback(async () => {
+    await Promise.all([loadControls(), loadMeta(), loadReviewCount()])
+  }, [loadControls, loadMeta, loadReviewCount])
 
   // CRUD handlers
   const handleCreate = async (data: typeof EMPTY_CONTROL) => {
@@ -154,7 +185,7 @@ export default function ControlLibraryPage() {
         alert(`Fehler: ${err.error || err.details || 'Unbekannt'}`)
         return
       }
-      await loadData()
+      await fullReload()
       setMode('list')
     } catch {
       alert('Netzwerkfehler')
@@ -177,7 +208,7 @@ export default function ControlLibraryPage() {
         alert(`Fehler: ${err.error || err.details || 'Unbekannt'}`)
         return
       }
-      await loadData()
+      await fullReload()
       setSelectedControl(null)
       setMode('list')
     } catch {
@@ -195,7 +226,7 @@ export default function ControlLibraryPage() {
         alert('Fehler beim Loeschen')
         return
       }
-      await loadData()
+      await fullReload()
       setSelectedControl(null)
       setMode('list')
     } catch {
@@ -211,11 +242,10 @@ export default function ControlLibraryPage() {
         body: JSON.stringify({ action }),
       })
       if (res.ok) {
-        await loadData()
+        await fullReload()
         if (reviewMode) {
-          const remaining = controls.filter(c =>
-            ['needs_review', 'too_close', 'duplicate'].includes(c.release_state) && c.control_id !== controlId
-          )
+          const remaining = reviewItems.filter(c => c.control_id !== controlId)
+          setReviewItems(remaining)
           if (remaining.length > 0) {
             const nextIdx = Math.min(reviewIndex, remaining.length - 1)
             setReviewIndex(nextIdx)
@@ -243,16 +273,25 @@ export default function ControlLibraryPage() {
     } catch { /* ignore */ }
   }
 
-  const enterReviewMode = () => {
-    if (reviewItems.length === 0) return
-    setReviewMode(true)
-    setReviewIndex(0)
-    setSelectedControl(reviewItems[0])
-    setMode('detail')
+  const enterReviewMode = async () => {
+    // Load review items from backend
+    try {
+      const res = await fetch(`${BACKEND_URL}?endpoint=controls&release_state=needs_review&limit=200`)
+      if (res.ok) {
+        const items = await res.json()
+        if (items.length > 0) {
+          setReviewItems(items)
+          setReviewMode(true)
+          setReviewIndex(0)
+          setSelectedControl(items[0])
+          setMode('detail')
+        }
+      }
+    } catch { /* ignore */ }
   }
 
   // Loading
-  if (loading) {
+  if (loading && controls.length === 0) {
     return (
       <div className="flex items-center justify-center h-96">
         <div className="animate-spin rounded-full h-8 w-8 border-2 border-purple-600 border-t-transparent" />
@@ -304,7 +343,7 @@ export default function ControlLibraryPage() {
         onEdit={() => setMode('edit')}
         onDelete={handleDelete}
         onReview={handleReview}
-        onRefresh={loadData}
+        onRefresh={fullReload}
         reviewMode={reviewMode}
         reviewIndex={reviewIndex}
         reviewTotal={reviewItems.length}
@@ -336,19 +375,18 @@ export default function ControlLibraryPage() {
             <div>
               <h1 className="text-lg font-semibold text-gray-900">Canonical Control Library</h1>
               <p className="text-xs text-gray-500">
-                {controls.length} unabhaengig formulierte Security Controls —{' '}
-                {controls.reduce((sum, c) => sum + c.open_anchors.length, 0)} Open-Source-Referenzen
+                {meta?.total ?? totalCount} Security Controls
               </p>
             </div>
           </div>
           <div className="flex items-center gap-2">
-            {reviewItems.length > 0 && (
+            {reviewCount > 0 && (
               <button
                 onClick={enterReviewMode}
                 className="flex items-center gap-1.5 px-3 py-2 text-sm text-white bg-yellow-600 rounded-lg hover:bg-yellow-700"
               >
                 <ListChecks className="w-4 h-4" />
-                Review ({reviewItems.length})
+                Review ({reviewCount})
               </button>
             )}
             <button
@@ -394,12 +432,19 @@ export default function ControlLibraryPage() {
               <Search className="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-gray-400" />
               <input
                 type="text"
-                placeholder="Controls durchsuchen..."
+                placeholder="Controls durchsuchen (ID, Titel, Objective)..."
                 value={searchQuery}
                 onChange={e => setSearchQuery(e.target.value)}
                 className="w-full pl-9 pr-4 py-2 text-sm border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-purple-500"
               />
             </div>
+            <button
+              onClick={() => { loadControls(); loadMeta(); loadReviewCount() }}
+              className="p-2 text-gray-400 hover:text-purple-600"
+              title="Aktualisieren"
+            >
+              <RefreshCw className={`w-4 h-4 ${loading ? 'animate-spin' : ''}`} />
+            </button>
           </div>
           <div className="flex items-center gap-2 flex-wrap">
             <Filter className="w-4 h-4 text-gray-400" />
@@ -420,8 +465,8 @@ export default function ControlLibraryPage() {
               className="text-sm border border-gray-300 rounded-lg px-2 py-1.5 focus:outline-none focus:ring-2 focus:ring-purple-500"
             >
               <option value="">Domain</option>
-              {domains.map(d => (
-                <option key={d} value={d}>{d}</option>
+              {(meta?.domains || []).map(d => (
+                <option key={d.domain} value={d.domain}>{d.domain} ({d.count})</option>
               ))}
             </select>
             <select
@@ -472,11 +517,23 @@ export default function ControlLibraryPage() {
               className="text-sm border border-gray-300 rounded-lg px-2 py-1.5 focus:outline-none focus:ring-2 focus:ring-purple-500 max-w-[220px]"
             >
               <option value="">Dokumentenursprung</option>
-              <option value="__none__">Ohne Quelle ({documentSources.noSourceCount})</option>
-              {documentSources.sources.map(([src, count]) => (
-                <option key={src} value={src}>{src} ({count})</option>
+              {meta && <option value="__none__">Ohne Quelle ({meta.no_source_count})</option>}
+              {(meta?.sources || []).map(s => (
+                <option key={s.source} value={s.source}>{s.source} ({s.count})</option>
               ))}
             </select>
+            <span className="text-gray-300 mx-1">|</span>
+            <ArrowUpDown className="w-4 h-4 text-gray-400" />
+            <select
+              value={sortBy}
+              onChange={e => setSortBy(e.target.value as 'id' | 'newest' | 'oldest' | 'source')}
+              className="text-sm border border-gray-300 rounded-lg px-2 py-1.5 focus:outline-none focus:ring-2 focus:ring-purple-500"
+            >
+              <option value="id">Sortierung: ID</option>
+              <option value="source">Nach Quelle</option>
+              <option value="newest">Neueste zuerst</option>
+              <option value="oldest">Aelteste zuerst</option>
+            </select>
           </div>
         </div>
 
@@ -504,15 +561,16 @@ export default function ControlLibraryPage() {
       {showGenerator && (
         <GeneratorModal
           onClose={() => setShowGenerator(false)}
-          onComplete={() => loadData()}
+          onComplete={() => fullReload()}
         />
       )}
 
       {/* Pagination Header */}
       <div className="px-6 py-2 bg-gray-50 border-b border-gray-200 flex items-center justify-between text-xs text-gray-500">
         <span>
-          {filteredControls.length} Controls gefunden
-          {filteredControls.length !== controls.length && ` (von ${controls.length} gesamt)`}
+          {totalCount} Controls gefunden
+          {totalCount !== (meta?.total ?? totalCount) && ` (von ${meta?.total} gesamt)`}
+          {loading && <span className="ml-2 text-purple-500">Lade...</span>}
         </span>
         <span>Seite {currentPage} von {totalPages}</span>
       </div>
@@ -520,9 +578,22 @@ export default function ControlLibraryPage() {
       {/* Control List */}
       <div className="flex-1 overflow-y-auto p-6">
         <div className="space-y-3">
-          {paginatedControls.map(ctrl => (
+          {controls.map((ctrl, idx) => {
+            // Show source group header when sorting by source
+            const prevSource = idx > 0 ? (controls[idx - 1].source_citation?.source || 'Ohne Quelle') : null
+            const curSource = ctrl.source_citation?.source || 'Ohne Quelle'
+            const showSourceHeader = sortBy === 'source' && curSource !== prevSource
+
+            return (
+            <div key={ctrl.control_id}>
+            {showSourceHeader && (
+              <div className="flex items-center gap-2 pt-3 pb-1">
+                <div className="h-px flex-1 bg-blue-200" />
+                <span className="text-xs font-semibold text-blue-700 bg-blue-50 px-2 py-0.5 rounded whitespace-nowrap">{curSource}</span>
+                <div className="h-px flex-1 bg-blue-200" />
+              </div>
+            )}
             <button
-              key={ctrl.control_id}
               onClick={() => { setSelectedControl(ctrl); setMode('detail') }}
               className="w-full text-left bg-white border border-gray-200 rounded-lg p-4 hover:border-purple-300 hover:shadow-sm transition-all group"
             >
@@ -536,6 +607,7 @@ export default function ControlLibraryPage() {
                     <VerificationMethodBadge method={ctrl.verification_method} />
                     <CategoryBadge category={ctrl.category} />
                     <TargetAudienceBadge audience={ctrl.target_audience} />
+                    <GenerationStrategyBadge strategy={ctrl.generation_strategy} />
                     {ctrl.risk_score !== null && (
                       <span className="text-xs text-gray-400">Score: {ctrl.risk_score}</span>
                     )}
@@ -543,7 +615,7 @@ export default function ControlLibraryPage() {
                   <h3 className="text-sm font-medium text-gray-900 group-hover:text-purple-700">{ctrl.title}</h3>
                   <p className="text-xs text-gray-500 mt-1 line-clamp-2">{ctrl.objective}</p>
 
-                  {/* Open anchors summary */}
+                  {/* Open anchors summary + timestamp */}
                   <div className="flex items-center gap-2 mt-2">
                     <BookOpen className="w-3 h-3 text-green-600" />
                     <span className="text-xs text-green-700">
@@ -555,16 +627,23 @@ export default function ControlLibraryPage() {
                         <span className="text-xs text-blue-600">{ctrl.source_citation.source}</span>
                       </>
                     )}
+                    <span className="text-gray-300">|</span>
+                    <Clock className="w-3 h-3 text-gray-400" />
+                    <span className="text-xs text-gray-400" title={ctrl.created_at}>
+                      {ctrl.created_at ? new Date(ctrl.created_at).toLocaleDateString('de-DE', { day: '2-digit', month: '2-digit', year: '2-digit', hour: '2-digit', minute: '2-digit' }) : '–'}
+                    </span>
                   </div>
                 </div>
                 <ChevronRight className="w-4 h-4 text-gray-300 group-hover:text-purple-500 flex-shrink-0 mt-1 ml-4" />
               </div>
             </button>
-          ))}
+            </div>
+            )
+          })}
 
-          {filteredControls.length === 0 && (
+          {controls.length === 0 && !loading && (
             <div className="text-center py-12 text-gray-400 text-sm">
-              {controls.length === 0
+              {totalCount === 0 && !debouncedSearch && !severityFilter && !domainFilter
                 ? 'Noch keine Controls vorhanden. Klicke auf "Neues Control" um zu starten.'
                 : 'Keine Controls gefunden.'}
             </div>