feat: add compliance modules 2-5 (dashboard, security templates, process manager, evidence collector)
All checks were successful
CI/CD / go-lint (push) Has been skipped
CI/CD / python-lint (push) Has been skipped
CI/CD / nodejs-lint (push) Has been skipped
CI/CD / test-go-ai-compliance (push) Successful in 32s
CI/CD / test-python-backend-compliance (push) Successful in 34s
CI/CD / test-python-document-crawler (push) Successful in 23s
CI/CD / test-python-dsms-gateway (push) Successful in 21s
CI/CD / validate-canonical-controls (push) Successful in 11s
CI/CD / Deploy (push) Successful in 2s

Module 2: Extended Compliance Dashboard with roadmap, module-status, next-actions, snapshots, score-history
Module 3: 7 German security document templates (IT-Sicherheitskonzept, Datenschutz, Backup, Logging, Incident-Response, Zugriff, Risikomanagement)
Module 4: Compliance Process Manager with CRUD, complete/skip/seed, ~50 seed tasks, 3-tab UI
Module 5: Evidence Collector Extended with automated checks, control-mapping, coverage report, 4-tab UI

Also includes: canonical control library enhancements (verification method, categories, dedup), control generator improvements, RAG client extensions

52 tests pass, frontend builds clean.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
2026-03-14 21:03:04 +01:00
parent 13d13c8226
commit 49ce417428
35 changed files with 8741 additions and 422 deletions

View File

@@ -443,6 +443,142 @@ func (c *LegalRAGClient) FormatLegalContextForPrompt(lc *LegalContext) string {
return buf.String()
}
// ScrollChunkResult represents a single chunk from the scroll/list endpoint.
type ScrollChunkResult struct {
ID string `json:"id"`
Text string `json:"text"`
RegulationCode string `json:"regulation_code"`
RegulationName string `json:"regulation_name"`
RegulationShort string `json:"regulation_short"`
Category string `json:"category"`
Article string `json:"article,omitempty"`
Paragraph string `json:"paragraph,omitempty"`
SourceURL string `json:"source_url,omitempty"`
}
// qdrantScrollRequest for the Qdrant scroll API.
type qdrantScrollRequest struct {
Limit int `json:"limit"`
Offset interface{} `json:"offset,omitempty"` // string (UUID) or null
WithPayload bool `json:"with_payload"`
WithVectors bool `json:"with_vectors"`
}
// qdrantScrollResponse from the Qdrant scroll API.
type qdrantScrollResponse struct {
Result struct {
Points []qdrantScrollPoint `json:"points"`
NextPageOffset interface{} `json:"next_page_offset"`
} `json:"result"`
}
type qdrantScrollPoint struct {
ID interface{} `json:"id"`
Payload map[string]interface{} `json:"payload"`
}
// ScrollChunks iterates over all chunks in a Qdrant collection using the scroll API.
// Pass an empty offset to start from the beginning. Returns chunks, next offset ID, and error.
func (c *LegalRAGClient) ScrollChunks(ctx context.Context, collection string, offset string, limit int) ([]ScrollChunkResult, string, error) {
scrollReq := qdrantScrollRequest{
Limit: limit,
WithPayload: true,
WithVectors: false,
}
if offset != "" {
// Qdrant expects integer point IDs — parse the offset string back to a number
// Try parsing as integer first, fall back to string (for UUID-based collections)
var offsetInt uint64
if _, err := fmt.Sscanf(offset, "%d", &offsetInt); err == nil {
scrollReq.Offset = offsetInt
} else {
scrollReq.Offset = offset
}
}
jsonBody, err := json.Marshal(scrollReq)
if err != nil {
return nil, "", fmt.Errorf("failed to marshal scroll request: %w", err)
}
url := fmt.Sprintf("%s/collections/%s/points/scroll", c.qdrantURL, collection)
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
if err != nil {
return nil, "", fmt.Errorf("failed to create scroll request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if c.qdrantAPIKey != "" {
req.Header.Set("api-key", c.qdrantAPIKey)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, "", fmt.Errorf("scroll request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, "", fmt.Errorf("qdrant returned %d: %s", resp.StatusCode, string(body))
}
var scrollResp qdrantScrollResponse
if err := json.NewDecoder(resp.Body).Decode(&scrollResp); err != nil {
return nil, "", fmt.Errorf("failed to decode scroll response: %w", err)
}
// Convert points to results
chunks := make([]ScrollChunkResult, len(scrollResp.Result.Points))
for i, pt := range scrollResp.Result.Points {
// Extract point ID as string
pointID := ""
if pt.ID != nil {
pointID = fmt.Sprintf("%v", pt.ID)
}
chunks[i] = ScrollChunkResult{
ID: pointID,
Text: getString(pt.Payload, "text"),
RegulationCode: getString(pt.Payload, "regulation_code"),
RegulationName: getString(pt.Payload, "regulation_name"),
RegulationShort: getString(pt.Payload, "regulation_short"),
Category: getString(pt.Payload, "category"),
Article: getString(pt.Payload, "article"),
Paragraph: getString(pt.Payload, "paragraph"),
SourceURL: getString(pt.Payload, "source_url"),
}
// Fallback: try alternate payload field names used in ingestion
if chunks[i].Text == "" {
chunks[i].Text = getString(pt.Payload, "chunk_text")
}
if chunks[i].RegulationCode == "" {
chunks[i].RegulationCode = getString(pt.Payload, "regulation_id")
}
if chunks[i].RegulationName == "" {
chunks[i].RegulationName = getString(pt.Payload, "regulation_name_de")
}
if chunks[i].SourceURL == "" {
chunks[i].SourceURL = getString(pt.Payload, "source")
}
}
// Extract next offset — Qdrant returns integer point IDs
nextOffset := ""
if scrollResp.Result.NextPageOffset != nil {
switch v := scrollResp.Result.NextPageOffset.(type) {
case float64:
nextOffset = fmt.Sprintf("%.0f", v)
case string:
nextOffset = v
default:
nextOffset = fmt.Sprintf("%v", v)
}
}
return chunks, nextOffset, nil
}
// Helper functions
func getString(m map[string]interface{}, key string) string {