From a228b3b528ad106e16f4677988635918981d7f23 Mon Sep 17 00:00:00 2001 From: Benjamin Admin Date: Mon, 2 Mar 2026 07:58:08 +0100 Subject: [PATCH] feat: add RAG corpus versioning and source policy backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part 1 — RAG Corpus Versioning: - New DB table compliance_corpus_versions (migration 017) - Go CorpusVersionStore with CRUD operations - Assessment struct extended with corpus_version_id - API endpoints: GET /rag/corpus-status, /rag/corpus-versions/:collection - RAG routes (search, regulations) now registered in main.go - Ingestion script registers corpus versions after each run - Frontend staleness badge in SDK sidebar Part 3 — Source Policy Backend: - New FastAPI router with CRUD for allowed sources, PII rules, operations matrix, audit trail, stats, and compliance report - SQLAlchemy models for all source policy tables (migration 001) - Frontend API base corrected from edu-search:8088/8089 to backend-compliance:8002/api Co-Authored-By: Claude Opus 4.6 --- .../app/(sdk)/sdk/source-policy/page.tsx | 12 +- .../components/sdk/Sidebar/SDKSidebar.tsx | 53 + admin-compliance/lib/sdk/context.tsx | 3 + admin-compliance/lib/sdk/types.ts | 19 + ai-compliance-sdk/cmd/server/main.go | 11 + .../internal/api/handlers/rag_handlers.go | 67 +- .../internal/ucca/corpus_version.go | 158 +++ ai-compliance-sdk/internal/ucca/models.go | 4 + ai-compliance-sdk/internal/ucca/store.go | 11 +- .../migrations/017_corpus_versioning.sql | 35 + .../compliance/api/source_policy_router.py | 503 +++++++++ .../compliance/db/source_policy_models.py | 105 ++ backend-compliance/main.py | 6 + .../migrations/001_source_policy.sql | 73 ++ scripts/ingest-legal-corpus.sh | 971 ++++++++++++++++++ 15 files changed, 2020 insertions(+), 11 deletions(-) create mode 100644 ai-compliance-sdk/internal/ucca/corpus_version.go create mode 100644 ai-compliance-sdk/migrations/017_corpus_versioning.sql create mode 100644 backend-compliance/compliance/api/source_policy_router.py create mode 100644 backend-compliance/compliance/db/source_policy_models.py create mode 100644 backend-compliance/migrations/001_source_policy.sql create mode 100755 scripts/ingest-legal-corpus.sh diff --git a/admin-compliance/app/(sdk)/sdk/source-policy/page.tsx b/admin-compliance/app/(sdk)/sdk/source-policy/page.tsx index 4efd3a7..04e5bcd 100644 --- a/admin-compliance/app/(sdk)/sdk/source-policy/page.tsx +++ b/admin-compliance/app/(sdk)/sdk/source-policy/page.tsx @@ -3,8 +3,8 @@ /** * Source Policy Management Page (SDK Version) * - * Whitelist-based data source management for edu-search-service. - * For auditors: Full audit trail for all changes. + * Whitelist-based data source management for compliance RAG corpus. + * Controls which legal sources may be used, PII rules, and audit trail. */ import { useState, useEffect } from 'react' @@ -15,14 +15,14 @@ import { OperationsMatrixTab } from '@/components/sdk/source-policy/OperationsMa import { PIIRulesTab } from '@/components/sdk/source-policy/PIIRulesTab' import { AuditTab } from '@/components/sdk/source-policy/AuditTab' -// API base URL for edu-search-service +// API base URL for backend-compliance const getApiBase = () => { - if (typeof window === 'undefined') return 'http://localhost:8088' + if (typeof window === 'undefined') return 'http://localhost:8002/api' const hostname = window.location.hostname if (hostname === 'localhost' || hostname === '127.0.0.1') { - return 'http://localhost:8088' + return 'http://localhost:8002/api' } - return `https://${hostname}:8089` + return `https://${hostname}:8002/api` } interface PolicyStats { diff --git a/admin-compliance/components/sdk/Sidebar/SDKSidebar.tsx b/admin-compliance/components/sdk/Sidebar/SDKSidebar.tsx index 3260c2e..fecd59d 100644 --- a/admin-compliance/components/sdk/Sidebar/SDKSidebar.tsx +++ b/admin-compliance/components/sdk/Sidebar/SDKSidebar.tsx @@ -10,6 +10,7 @@ import { getStepsForPackage, type SDKPackageId, type SDKStep, + type RAGCorpusStatus, } from '@/lib/sdk' // ============================================================================= @@ -288,6 +289,41 @@ interface SDKSidebarProps { onCollapsedChange?: (collapsed: boolean) => void } +// ============================================================================= +// CORPUS STALENESS INFO +// ============================================================================= + +function CorpusStalenessInfo({ ragCorpusStatus }: { ragCorpusStatus: RAGCorpusStatus }) { + const collections = ragCorpusStatus.collections + const collectionNames = Object.keys(collections) + if (collectionNames.length === 0) return null + + // Check if corpus was updated after the last fetch (simplified: show last update time) + const lastUpdated = collectionNames.reduce((latest, name) => { + const updated = new Date(collections[name].last_updated) + return updated > latest ? updated : latest + }, new Date(0)) + + const daysSinceUpdate = Math.floor((Date.now() - lastUpdated.getTime()) / (1000 * 60 * 60 * 24)) + const totalChunks = collectionNames.reduce((sum, name) => sum + collections[name].chunks_count, 0) + + return ( +
+
+
30 ? 'bg-amber-400' : 'bg-green-400'}`} /> + + RAG Corpus: {totalChunks} Chunks + +
+ {daysSinceUpdate > 30 && ( +
+ Corpus {daysSinceUpdate}d alt — Re-Evaluation empfohlen +
+ )} +
+ ) +} + export function SDKSidebar({ collapsed = false, onCollapsedChange }: SDKSidebarProps) { const pathname = usePathname() const { state, packageCompletion, completionPercentage, getCheckpointStatus } = useSDK() @@ -391,6 +427,11 @@ export function SDKSidebar({ collapsed = false, onCollapsedChange }: SDKSidebarP
)} + {/* RAG Corpus Staleness Badge */} + {!collapsed && state.ragCorpusStatus && ( + + )} + {/* Navigation - 5 Packages */}