feat: add compliance modules 2-5 (dashboard, security templates, process manager, evidence collector)
All checks were successful
CI/CD / go-lint (push) Has been skipped
CI/CD / python-lint (push) Has been skipped
CI/CD / nodejs-lint (push) Has been skipped
CI/CD / test-go-ai-compliance (push) Successful in 32s
CI/CD / test-python-backend-compliance (push) Successful in 34s
CI/CD / test-python-document-crawler (push) Successful in 23s
CI/CD / test-python-dsms-gateway (push) Successful in 21s
CI/CD / validate-canonical-controls (push) Successful in 11s
CI/CD / Deploy (push) Successful in 2s
All checks were successful
CI/CD / go-lint (push) Has been skipped
CI/CD / python-lint (push) Has been skipped
CI/CD / nodejs-lint (push) Has been skipped
CI/CD / test-go-ai-compliance (push) Successful in 32s
CI/CD / test-python-backend-compliance (push) Successful in 34s
CI/CD / test-python-document-crawler (push) Successful in 23s
CI/CD / test-python-dsms-gateway (push) Successful in 21s
CI/CD / validate-canonical-controls (push) Successful in 11s
CI/CD / Deploy (push) Successful in 2s
Module 2: Extended Compliance Dashboard with roadmap, module-status, next-actions, snapshots, score-history Module 3: 7 German security document templates (IT-Sicherheitskonzept, Datenschutz, Backup, Logging, Incident-Response, Zugriff, Risikomanagement) Module 4: Compliance Process Manager with CRUD, complete/skip/seed, ~50 seed tasks, 3-tab UI Module 5: Evidence Collector Extended with automated checks, control-mapping, coverage report, 4-tab UI Also includes: canonical control library enhancements (verification method, categories, dedup), control generator improvements, RAG client extensions 52 tests pass, frontend builds clean. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -443,6 +443,142 @@ func (c *LegalRAGClient) FormatLegalContextForPrompt(lc *LegalContext) string {
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// ScrollChunkResult represents a single chunk from the scroll/list endpoint.
|
||||
type ScrollChunkResult struct {
|
||||
ID string `json:"id"`
|
||||
Text string `json:"text"`
|
||||
RegulationCode string `json:"regulation_code"`
|
||||
RegulationName string `json:"regulation_name"`
|
||||
RegulationShort string `json:"regulation_short"`
|
||||
Category string `json:"category"`
|
||||
Article string `json:"article,omitempty"`
|
||||
Paragraph string `json:"paragraph,omitempty"`
|
||||
SourceURL string `json:"source_url,omitempty"`
|
||||
}
|
||||
|
||||
// qdrantScrollRequest for the Qdrant scroll API.
|
||||
type qdrantScrollRequest struct {
|
||||
Limit int `json:"limit"`
|
||||
Offset interface{} `json:"offset,omitempty"` // string (UUID) or null
|
||||
WithPayload bool `json:"with_payload"`
|
||||
WithVectors bool `json:"with_vectors"`
|
||||
}
|
||||
|
||||
// qdrantScrollResponse from the Qdrant scroll API.
|
||||
type qdrantScrollResponse struct {
|
||||
Result struct {
|
||||
Points []qdrantScrollPoint `json:"points"`
|
||||
NextPageOffset interface{} `json:"next_page_offset"`
|
||||
} `json:"result"`
|
||||
}
|
||||
|
||||
type qdrantScrollPoint struct {
|
||||
ID interface{} `json:"id"`
|
||||
Payload map[string]interface{} `json:"payload"`
|
||||
}
|
||||
|
||||
// ScrollChunks iterates over all chunks in a Qdrant collection using the scroll API.
|
||||
// Pass an empty offset to start from the beginning. Returns chunks, next offset ID, and error.
|
||||
func (c *LegalRAGClient) ScrollChunks(ctx context.Context, collection string, offset string, limit int) ([]ScrollChunkResult, string, error) {
|
||||
scrollReq := qdrantScrollRequest{
|
||||
Limit: limit,
|
||||
WithPayload: true,
|
||||
WithVectors: false,
|
||||
}
|
||||
if offset != "" {
|
||||
// Qdrant expects integer point IDs — parse the offset string back to a number
|
||||
// Try parsing as integer first, fall back to string (for UUID-based collections)
|
||||
var offsetInt uint64
|
||||
if _, err := fmt.Sscanf(offset, "%d", &offsetInt); err == nil {
|
||||
scrollReq.Offset = offsetInt
|
||||
} else {
|
||||
scrollReq.Offset = offset
|
||||
}
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(scrollReq)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to marshal scroll request: %w", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/collections/%s/points/scroll", c.qdrantURL, collection)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to create scroll request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if c.qdrantAPIKey != "" {
|
||||
req.Header.Set("api-key", c.qdrantAPIKey)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("scroll request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, "", fmt.Errorf("qdrant returned %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var scrollResp qdrantScrollResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&scrollResp); err != nil {
|
||||
return nil, "", fmt.Errorf("failed to decode scroll response: %w", err)
|
||||
}
|
||||
|
||||
// Convert points to results
|
||||
chunks := make([]ScrollChunkResult, len(scrollResp.Result.Points))
|
||||
for i, pt := range scrollResp.Result.Points {
|
||||
// Extract point ID as string
|
||||
pointID := ""
|
||||
if pt.ID != nil {
|
||||
pointID = fmt.Sprintf("%v", pt.ID)
|
||||
}
|
||||
|
||||
chunks[i] = ScrollChunkResult{
|
||||
ID: pointID,
|
||||
Text: getString(pt.Payload, "text"),
|
||||
RegulationCode: getString(pt.Payload, "regulation_code"),
|
||||
RegulationName: getString(pt.Payload, "regulation_name"),
|
||||
RegulationShort: getString(pt.Payload, "regulation_short"),
|
||||
Category: getString(pt.Payload, "category"),
|
||||
Article: getString(pt.Payload, "article"),
|
||||
Paragraph: getString(pt.Payload, "paragraph"),
|
||||
SourceURL: getString(pt.Payload, "source_url"),
|
||||
}
|
||||
|
||||
// Fallback: try alternate payload field names used in ingestion
|
||||
if chunks[i].Text == "" {
|
||||
chunks[i].Text = getString(pt.Payload, "chunk_text")
|
||||
}
|
||||
if chunks[i].RegulationCode == "" {
|
||||
chunks[i].RegulationCode = getString(pt.Payload, "regulation_id")
|
||||
}
|
||||
if chunks[i].RegulationName == "" {
|
||||
chunks[i].RegulationName = getString(pt.Payload, "regulation_name_de")
|
||||
}
|
||||
if chunks[i].SourceURL == "" {
|
||||
chunks[i].SourceURL = getString(pt.Payload, "source")
|
||||
}
|
||||
}
|
||||
|
||||
// Extract next offset — Qdrant returns integer point IDs
|
||||
nextOffset := ""
|
||||
if scrollResp.Result.NextPageOffset != nil {
|
||||
switch v := scrollResp.Result.NextPageOffset.(type) {
|
||||
case float64:
|
||||
nextOffset = fmt.Sprintf("%.0f", v)
|
||||
case string:
|
||||
nextOffset = v
|
||||
default:
|
||||
nextOffset = fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
|
||||
return chunks, nextOffset, nil
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
func getString(m map[string]interface{}, key string) string {
|
||||
|
||||
Reference in New Issue
Block a user