feat: add RAG corpus versioning and source policy backend
All checks were successful
CI / go-lint (push) Has been skipped
CI / python-lint (push) Has been skipped
CI / nodejs-lint (push) Has been skipped
CI / test-go-ai-compliance (push) Successful in 34s
CI / test-python-backend-compliance (push) Successful in 32s
CI / test-python-document-crawler (push) Successful in 23s
CI / test-python-dsms-gateway (push) Successful in 18s

Part 1 — RAG Corpus Versioning:
- New DB table compliance_corpus_versions (migration 017)
- Go CorpusVersionStore with CRUD operations
- Assessment struct extended with corpus_version_id
- API endpoints: GET /rag/corpus-status, /rag/corpus-versions/:collection
- RAG routes (search, regulations) now registered in main.go
- Ingestion script registers corpus versions after each run
- Frontend staleness badge in SDK sidebar

Part 3 — Source Policy Backend:
- New FastAPI router with CRUD for allowed sources, PII rules,
  operations matrix, audit trail, stats, and compliance report
- SQLAlchemy models for all source policy tables (migration 001)
- Frontend API base corrected from edu-search:8088/8089 to
  backend-compliance:8002/api

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-02 07:58:08 +01:00
parent 187dbf1b77
commit a228b3b528
15 changed files with 2020 additions and 11 deletions

View File

@@ -3,8 +3,8 @@
/**
* Source Policy Management Page (SDK Version)
*
* Whitelist-based data source management for edu-search-service.
* For auditors: Full audit trail for all changes.
* Whitelist-based data source management for compliance RAG corpus.
* Controls which legal sources may be used, PII rules, and audit trail.
*/
import { useState, useEffect } from 'react'
@@ -15,14 +15,14 @@ import { OperationsMatrixTab } from '@/components/sdk/source-policy/OperationsMa
import { PIIRulesTab } from '@/components/sdk/source-policy/PIIRulesTab'
import { AuditTab } from '@/components/sdk/source-policy/AuditTab'
// API base URL for edu-search-service
// API base URL for backend-compliance
const getApiBase = () => {
if (typeof window === 'undefined') return 'http://localhost:8088'
if (typeof window === 'undefined') return 'http://localhost:8002/api'
const hostname = window.location.hostname
if (hostname === 'localhost' || hostname === '127.0.0.1') {
return 'http://localhost:8088'
return 'http://localhost:8002/api'
}
return `https://${hostname}:8089`
return `https://${hostname}:8002/api`
}
interface PolicyStats {

View File

@@ -10,6 +10,7 @@ import {
getStepsForPackage,
type SDKPackageId,
type SDKStep,
type RAGCorpusStatus,
} from '@/lib/sdk'
// =============================================================================
@@ -288,6 +289,41 @@ interface SDKSidebarProps {
onCollapsedChange?: (collapsed: boolean) => void
}
// =============================================================================
// CORPUS STALENESS INFO
// =============================================================================
function CorpusStalenessInfo({ ragCorpusStatus }: { ragCorpusStatus: RAGCorpusStatus }) {
const collections = ragCorpusStatus.collections
const collectionNames = Object.keys(collections)
if (collectionNames.length === 0) return null
// Check if corpus was updated after the last fetch (simplified: show last update time)
const lastUpdated = collectionNames.reduce((latest, name) => {
const updated = new Date(collections[name].last_updated)
return updated > latest ? updated : latest
}, new Date(0))
const daysSinceUpdate = Math.floor((Date.now() - lastUpdated.getTime()) / (1000 * 60 * 60 * 24))
const totalChunks = collectionNames.reduce((sum, name) => sum + collections[name].chunks_count, 0)
return (
<div className="px-4 py-2 border-b border-gray-100">
<div className="flex items-center gap-2 text-xs">
<div className={`w-2 h-2 rounded-full flex-shrink-0 ${daysSinceUpdate > 30 ? 'bg-amber-400' : 'bg-green-400'}`} />
<span className="text-gray-500 truncate">
RAG Corpus: {totalChunks} Chunks
</span>
</div>
{daysSinceUpdate > 30 && (
<div className="mt-1 text-xs text-amber-600 bg-amber-50 rounded px-2 py-1">
Corpus {daysSinceUpdate}d alt Re-Evaluation empfohlen
</div>
)}
</div>
)
}
export function SDKSidebar({ collapsed = false, onCollapsedChange }: SDKSidebarProps) {
const pathname = usePathname()
const { state, packageCompletion, completionPercentage, getCheckpointStatus } = useSDK()
@@ -391,6 +427,11 @@ export function SDKSidebar({ collapsed = false, onCollapsedChange }: SDKSidebarP
</div>
)}
{/* RAG Corpus Staleness Badge */}
{!collapsed && state.ragCorpusStatus && (
<CorpusStalenessInfo ragCorpusStatus={state.ragCorpusStatus} />
)}
{/* Navigation - 5 Packages */}
<nav className="flex-1 overflow-y-auto">
{SDK_PACKAGES.map(pkg => {
@@ -510,6 +551,18 @@ export function SDKSidebar({ collapsed = false, onCollapsedChange }: SDKSidebarP
isActive={pathname === '/sdk/dsms'}
collapsed={collapsed}
/>
<AdditionalModuleItem
href="/development/sdk-flow"
icon={
<svg className="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2}
d="M13 10V3L4 14h7v7l9-11h-7z" />
</svg>
}
label="SDK Flow"
isActive={pathname === '/development/sdk-flow'}
collapsed={collapsed}
/>
{state.companyProfile?.machineBuilder?.ceMarkingRequired && (
<AdditionalModuleItem
href="/sdk/iace"

View File

@@ -102,6 +102,9 @@ const initialState: SDKState = {
// IACE (Industrial AI Compliance Engine)
iaceProjects: [],
// RAG Corpus Versioning
ragCorpusStatus: null,
// Security
sbom: null,
securityIssues: [],

View File

@@ -975,6 +975,22 @@ export interface SBOMDependency {
to: string
}
// RAG Corpus Versioning
export interface RAGCorpusCollectionStatus {
id: string
current_version: string
documents_count: number
chunks_count: number
regulations: string[]
last_updated: string
digest: string
}
export interface RAGCorpusStatus {
collections: Record<string, RAGCorpusCollectionStatus>
fetchedAt: string
}
export interface SBOM {
format: 'CycloneDX' | 'SPDX'
version: string
@@ -1504,6 +1520,9 @@ export interface SDKState {
// IACE (Industrial AI Compliance Engine)
iaceProjects: IACEProjectSummary[]
// RAG Corpus Versioning
ragCorpusStatus: RAGCorpusStatus | null
// Security
sbom: SBOM | null
securityIssues: SecurityIssue[]