merge: Resolve conflicts with gitea remote
Keep local versions for .gitignore, vendors route (full header forwarding), investor-agent soul (slide-awareness + follow-up questions), mkdocs site_url. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
31
admin-v2/.docker/build-ci-images.sh
Executable file
31
admin-v2/.docker/build-ci-images.sh
Executable file
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# Build CI Docker Images for BreakPilot
|
||||
# Run this script on the Mac Mini to build the custom CI images
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
echo "=== Building BreakPilot CI Images ==="
|
||||
echo "Project directory: $PROJECT_DIR"
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# Build Python CI image with WeasyPrint
|
||||
echo ""
|
||||
echo "Building breakpilot/python-ci:3.12 ..."
|
||||
docker build \
|
||||
-t breakpilot/python-ci:3.12 \
|
||||
-t breakpilot/python-ci:latest \
|
||||
-f .docker/python-ci.Dockerfile \
|
||||
.
|
||||
|
||||
echo ""
|
||||
echo "=== Build complete ==="
|
||||
echo ""
|
||||
echo "Images built:"
|
||||
docker images | grep breakpilot/python-ci
|
||||
|
||||
echo ""
|
||||
echo "To use in Woodpecker CI, the image is already configured in .woodpecker/main.yml"
|
||||
51
admin-v2/.docker/python-ci.Dockerfile
Normal file
51
admin-v2/.docker/python-ci.Dockerfile
Normal file
@@ -0,0 +1,51 @@
|
||||
# Custom Python CI Image with WeasyPrint Dependencies
|
||||
# Build: docker build -t breakpilot/python-ci:3.12 -f .docker/python-ci.Dockerfile .
|
||||
#
|
||||
# This image includes all system libraries needed for:
|
||||
# - WeasyPrint (PDF generation)
|
||||
# - psycopg2 (PostgreSQL)
|
||||
# - General Python testing
|
||||
|
||||
FROM python:3.12-slim
|
||||
|
||||
LABEL maintainer="BreakPilot Team"
|
||||
LABEL description="Python 3.12 with WeasyPrint and test dependencies for CI"
|
||||
|
||||
# Install system dependencies in a single layer
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
# WeasyPrint dependencies
|
||||
libpango-1.0-0 \
|
||||
libpangocairo-1.0-0 \
|
||||
libpangoft2-1.0-0 \
|
||||
libgdk-pixbuf-2.0-0 \
|
||||
libffi-dev \
|
||||
libcairo2 \
|
||||
libcairo2-dev \
|
||||
libgirepository1.0-dev \
|
||||
gir1.2-pango-1.0 \
|
||||
# PostgreSQL client (for psycopg2)
|
||||
libpq-dev \
|
||||
# Build tools (for some pip packages)
|
||||
gcc \
|
||||
g++ \
|
||||
# Useful utilities
|
||||
curl \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& apt-get clean
|
||||
|
||||
# Pre-install commonly used Python packages for faster CI
|
||||
RUN pip install --no-cache-dir \
|
||||
pytest \
|
||||
pytest-cov \
|
||||
pytest-asyncio \
|
||||
pytest-json-report \
|
||||
psycopg2-binary \
|
||||
weasyprint \
|
||||
httpx
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Default command
|
||||
CMD ["python", "--version"]
|
||||
124
admin-v2/.env.example
Normal file
124
admin-v2/.env.example
Normal file
@@ -0,0 +1,124 @@
|
||||
# BreakPilot PWA - Environment Configuration
|
||||
# Kopieren Sie diese Datei nach .env und passen Sie die Werte an
|
||||
|
||||
# ================================================
|
||||
# Allgemein
|
||||
# ================================================
|
||||
ENVIRONMENT=development
|
||||
# ENVIRONMENT=production
|
||||
|
||||
# ================================================
|
||||
# Sicherheit
|
||||
# ================================================
|
||||
# WICHTIG: In Produktion sichere Schluessel verwenden!
|
||||
# Generieren mit: openssl rand -hex 32
|
||||
JWT_SECRET=CHANGE_ME_RUN_openssl_rand_hex_32
|
||||
JWT_REFRESH_SECRET=CHANGE_ME_RUN_openssl_rand_hex_32
|
||||
|
||||
# ================================================
|
||||
# Keycloak (Optional - fuer Produktion empfohlen)
|
||||
# ================================================
|
||||
# Wenn Keycloak konfiguriert ist, wird es fuer Authentifizierung verwendet.
|
||||
# Ohne Keycloak wird lokales JWT verwendet (gut fuer Entwicklung).
|
||||
#
|
||||
# KEYCLOAK_SERVER_URL=https://keycloak.breakpilot.app
|
||||
# KEYCLOAK_REALM=breakpilot
|
||||
# KEYCLOAK_CLIENT_ID=breakpilot-backend
|
||||
# KEYCLOAK_CLIENT_SECRET=your-client-secret
|
||||
# KEYCLOAK_VERIFY_SSL=true
|
||||
|
||||
# ================================================
|
||||
# E-Mail Konfiguration
|
||||
# ================================================
|
||||
|
||||
# === ENTWICKLUNG (Mailpit - Standardwerte) ===
|
||||
# Mailpit fängt alle E-Mails ab und zeigt sie unter http://localhost:8025
|
||||
SMTP_HOST=mailpit
|
||||
SMTP_PORT=1025
|
||||
SMTP_USERNAME=
|
||||
SMTP_PASSWORD=
|
||||
SMTP_FROM_NAME=BreakPilot
|
||||
SMTP_FROM_ADDR=noreply@breakpilot.app
|
||||
FRONTEND_URL=http://localhost:8000
|
||||
|
||||
# === PRODUKTION (Beispiel für verschiedene Provider) ===
|
||||
|
||||
# --- Option 1: Eigener Mailserver ---
|
||||
# SMTP_HOST=mail.ihredomain.de
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USERNAME=noreply@ihredomain.de
|
||||
# SMTP_PASSWORD=ihr-sicheres-passwort
|
||||
# SMTP_FROM_NAME=BreakPilot
|
||||
# SMTP_FROM_ADDR=noreply@ihredomain.de
|
||||
# FRONTEND_URL=https://app.ihredomain.de
|
||||
|
||||
# --- Option 2: SendGrid ---
|
||||
# SMTP_HOST=smtp.sendgrid.net
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USERNAME=apikey
|
||||
# SMTP_PASSWORD=SG.xxxxxxxxxxxxxxxxxxxxx
|
||||
# SMTP_FROM_NAME=BreakPilot
|
||||
# SMTP_FROM_ADDR=noreply@ihredomain.de
|
||||
|
||||
# --- Option 3: Mailgun ---
|
||||
# SMTP_HOST=smtp.mailgun.org
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USERNAME=postmaster@mg.ihredomain.de
|
||||
# SMTP_PASSWORD=ihr-mailgun-passwort
|
||||
# SMTP_FROM_NAME=BreakPilot
|
||||
# SMTP_FROM_ADDR=noreply@mg.ihredomain.de
|
||||
|
||||
# --- Option 4: Amazon SES ---
|
||||
# SMTP_HOST=email-smtp.eu-central-1.amazonaws.com
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USERNAME=AKIAXXXXXXXXXXXXXXXX
|
||||
# SMTP_PASSWORD=ihr-ses-secret
|
||||
# SMTP_FROM_NAME=BreakPilot
|
||||
# SMTP_FROM_ADDR=noreply@ihredomain.de
|
||||
|
||||
# ================================================
|
||||
# Datenbank
|
||||
# ================================================
|
||||
POSTGRES_USER=breakpilot
|
||||
POSTGRES_PASSWORD=breakpilot123
|
||||
POSTGRES_DB=breakpilot_db
|
||||
DATABASE_URL=postgres://breakpilot:breakpilot123@localhost:5432/breakpilot_db?sslmode=disable
|
||||
|
||||
# ================================================
|
||||
# Optional: AI Integration
|
||||
# ================================================
|
||||
# ANTHROPIC_API_KEY=your-anthropic-api-key-here
|
||||
|
||||
# ================================================
|
||||
# Breakpilot Drive - Lernspiel
|
||||
# ================================================
|
||||
# Aktiviert Datenbank-Speicherung fuer Spielsessions
|
||||
GAME_USE_DATABASE=true
|
||||
|
||||
# LLM fuer Quiz-Fragen-Generierung (optional)
|
||||
# Wenn nicht gesetzt, werden statische Fragen verwendet
|
||||
GAME_LLM_MODEL=llama-3.1-8b
|
||||
GAME_LLM_FALLBACK_MODEL=claude-3-haiku
|
||||
|
||||
# Feature Flags
|
||||
GAME_REQUIRE_AUTH=false
|
||||
GAME_REQUIRE_BILLING=false
|
||||
GAME_ENABLE_LEADERBOARDS=true
|
||||
|
||||
# Task-Kosten fuer Billing (wenn aktiviert)
|
||||
GAME_SESSION_TASK_COST=1.0
|
||||
GAME_QUICK_SESSION_TASK_COST=0.5
|
||||
|
||||
# ================================================
|
||||
# Woodpecker CI/CD
|
||||
# ================================================
|
||||
# URL zum Woodpecker Server
|
||||
WOODPECKER_URL=http://woodpecker-server:8000
|
||||
# API Token für Dashboard-Integration (Pipeline-Start)
|
||||
# Erstellen unter: http://macmini:8090 → User Settings → Personal Access Tokens
|
||||
WOODPECKER_TOKEN=
|
||||
|
||||
# ================================================
|
||||
# Debug
|
||||
# ================================================
|
||||
DEBUG=false
|
||||
132
admin-v2/.github/dependabot.yml
vendored
Normal file
132
admin-v2/.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,132 @@
|
||||
# Dependabot Configuration for BreakPilot PWA
|
||||
# This file configures Dependabot to automatically check for outdated dependencies
|
||||
# and create pull requests to update them
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
# Go dependencies (consent-service)
|
||||
- package-ecosystem: "gomod"
|
||||
directory: "/consent-service"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "06:00"
|
||||
timezone: "Europe/Berlin"
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "go"
|
||||
- "security"
|
||||
commit-message:
|
||||
prefix: "deps(go):"
|
||||
groups:
|
||||
go-minor:
|
||||
patterns:
|
||||
- "*"
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
|
||||
# Python dependencies (backend)
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/backend"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "06:00"
|
||||
timezone: "Europe/Berlin"
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "python"
|
||||
- "security"
|
||||
commit-message:
|
||||
prefix: "deps(python):"
|
||||
groups:
|
||||
python-minor:
|
||||
patterns:
|
||||
- "*"
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
|
||||
# Node.js dependencies (website)
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/website"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "06:00"
|
||||
timezone: "Europe/Berlin"
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "javascript"
|
||||
- "security"
|
||||
commit-message:
|
||||
prefix: "deps(npm):"
|
||||
groups:
|
||||
npm-minor:
|
||||
patterns:
|
||||
- "*"
|
||||
update-types:
|
||||
- "minor"
|
||||
- "patch"
|
||||
|
||||
# GitHub Actions
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "06:00"
|
||||
timezone: "Europe/Berlin"
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "github-actions"
|
||||
commit-message:
|
||||
prefix: "deps(actions):"
|
||||
|
||||
# Docker base images
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/consent-service"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "06:00"
|
||||
timezone: "Europe/Berlin"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
- "security"
|
||||
commit-message:
|
||||
prefix: "deps(docker):"
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/backend"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "06:00"
|
||||
timezone: "Europe/Berlin"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
- "security"
|
||||
commit-message:
|
||||
prefix: "deps(docker):"
|
||||
|
||||
- package-ecosystem: "docker"
|
||||
directory: "/website"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "06:00"
|
||||
timezone: "Europe/Berlin"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "docker"
|
||||
- "security"
|
||||
commit-message:
|
||||
prefix: "deps(docker):"
|
||||
503
admin-v2/.github/workflows/ci.yml
vendored
Normal file
503
admin-v2/.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,503 @@
|
||||
name: CI/CD Pipeline
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop]
|
||||
pull_request:
|
||||
branches: [main, develop]
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.21'
|
||||
PYTHON_VERSION: '3.11'
|
||||
NODE_VERSION: '20'
|
||||
POSTGRES_USER: breakpilot
|
||||
POSTGRES_PASSWORD: breakpilot123
|
||||
POSTGRES_DB: breakpilot_test
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_PREFIX: ${{ github.repository_owner }}/breakpilot
|
||||
|
||||
jobs:
|
||||
# ==========================================
|
||||
# Go Consent Service Tests
|
||||
# ==========================================
|
||||
go-tests:
|
||||
name: Go Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: ${{ env.POSTGRES_USER }}
|
||||
POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
|
||||
POSTGRES_DB: ${{ env.POSTGRES_DB }}
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
cache-dependency-path: consent-service/go.sum
|
||||
|
||||
- name: Download dependencies
|
||||
working-directory: ./consent-service
|
||||
run: go mod download
|
||||
|
||||
- name: Run Go Vet
|
||||
working-directory: ./consent-service
|
||||
run: go vet ./...
|
||||
|
||||
- name: Run Unit Tests
|
||||
working-directory: ./consent-service
|
||||
run: go test -v -race -coverprofile=coverage.out ./...
|
||||
env:
|
||||
DATABASE_URL: postgres://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@localhost:5432/${{ env.POSTGRES_DB }}?sslmode=disable
|
||||
JWT_SECRET: test-jwt-secret-for-ci
|
||||
JWT_REFRESH_SECRET: test-refresh-secret-for-ci
|
||||
|
||||
- name: Check Coverage
|
||||
working-directory: ./consent-service
|
||||
run: |
|
||||
go tool cover -func=coverage.out
|
||||
COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${COVERAGE}%"
|
||||
if (( $(echo "$COVERAGE < 50" | bc -l) )); then
|
||||
echo "::warning::Coverage is below 50%"
|
||||
fi
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
files: ./consent-service/coverage.out
|
||||
flags: go
|
||||
name: go-coverage
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Python Backend Tests
|
||||
# ==========================================
|
||||
python-tests:
|
||||
name: Python Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: 'pip'
|
||||
cache-dependency-path: backend/requirements.txt
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./backend
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install pytest pytest-cov pytest-asyncio httpx
|
||||
|
||||
- name: Run Python Tests
|
||||
working-directory: ./backend
|
||||
run: pytest -v --cov=. --cov-report=xml --cov-report=term-missing
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
files: ./backend/coverage.xml
|
||||
flags: python
|
||||
name: python-coverage
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Node.js Website Tests
|
||||
# ==========================================
|
||||
website-tests:
|
||||
name: Website Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: 'npm'
|
||||
cache-dependency-path: website/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./website
|
||||
run: npm ci
|
||||
|
||||
- name: Run TypeScript check
|
||||
working-directory: ./website
|
||||
run: npx tsc --noEmit
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run ESLint
|
||||
working-directory: ./website
|
||||
run: npm run lint
|
||||
continue-on-error: true
|
||||
|
||||
- name: Build website
|
||||
working-directory: ./website
|
||||
run: npm run build
|
||||
env:
|
||||
NEXT_PUBLIC_BILLING_API_URL: http://localhost:8083
|
||||
NEXT_PUBLIC_APP_URL: http://localhost:3000
|
||||
|
||||
# ==========================================
|
||||
# Linting
|
||||
# ==========================================
|
||||
lint:
|
||||
name: Linting
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@v4
|
||||
with:
|
||||
version: latest
|
||||
working-directory: ./consent-service
|
||||
args: --timeout=5m
|
||||
continue-on-error: true
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install Python linters
|
||||
run: pip install flake8 black isort
|
||||
|
||||
- name: Run flake8
|
||||
working-directory: ./backend
|
||||
run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
continue-on-error: true
|
||||
|
||||
- name: Check Black formatting
|
||||
working-directory: ./backend
|
||||
run: black --check --diff .
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Security Scan
|
||||
# ==========================================
|
||||
security:
|
||||
name: Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run Trivy vulnerability scanner
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
scan-type: 'fs'
|
||||
scan-ref: '.'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
exit-code: '0'
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run Go security check
|
||||
uses: securego/gosec@master
|
||||
with:
|
||||
args: '-no-fail -fmt sarif -out results.sarif ./consent-service/...'
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Docker Build & Push
|
||||
# ==========================================
|
||||
docker-build:
|
||||
name: Docker Build & Push
|
||||
runs-on: ubuntu-latest
|
||||
needs: [go-tests, python-tests, website-tests]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata for consent-service
|
||||
id: meta-consent
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-consent-service
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build and push consent-service
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./consent-service
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta-consent.outputs.tags }}
|
||||
labels: ${{ steps.meta-consent.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Extract metadata for backend
|
||||
id: meta-backend
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-backend
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build and push backend
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./backend
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta-backend.outputs.tags }}
|
||||
labels: ${{ steps.meta-backend.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Extract metadata for website
|
||||
id: meta-website
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-website
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build and push website
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ./website
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta-website.outputs.tags }}
|
||||
labels: ${{ steps.meta-website.outputs.labels }}
|
||||
build-args: |
|
||||
NEXT_PUBLIC_BILLING_API_URL=${{ vars.NEXT_PUBLIC_BILLING_API_URL || 'http://localhost:8083' }}
|
||||
NEXT_PUBLIC_APP_URL=${{ vars.NEXT_PUBLIC_APP_URL || 'http://localhost:3000' }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
# ==========================================
|
||||
# Integration Tests
|
||||
# ==========================================
|
||||
integration-tests:
|
||||
name: Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
needs: [docker-build]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Start services with Docker Compose
|
||||
run: |
|
||||
docker compose up -d postgres mailpit
|
||||
sleep 10
|
||||
|
||||
- name: Run consent-service
|
||||
working-directory: ./consent-service
|
||||
run: |
|
||||
go build -o consent-service ./cmd/server
|
||||
./consent-service &
|
||||
sleep 5
|
||||
env:
|
||||
DATABASE_URL: postgres://breakpilot:breakpilot123@localhost:5432/breakpilot_db?sslmode=disable
|
||||
JWT_SECRET: test-jwt-secret
|
||||
JWT_REFRESH_SECRET: test-refresh-secret
|
||||
SMTP_HOST: localhost
|
||||
SMTP_PORT: 1025
|
||||
|
||||
- name: Health Check
|
||||
run: |
|
||||
curl -f http://localhost:8081/health || exit 1
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
# Test Auth endpoints
|
||||
curl -s http://localhost:8081/api/v1/auth/health
|
||||
|
||||
# Test Document endpoints
|
||||
curl -s http://localhost:8081/api/v1/documents
|
||||
continue-on-error: true
|
||||
|
||||
- name: Stop services
|
||||
if: always()
|
||||
run: docker compose down
|
||||
|
||||
# ==========================================
|
||||
# Deploy to Staging
|
||||
# ==========================================
|
||||
deploy-staging:
|
||||
name: Deploy to Staging
|
||||
runs-on: ubuntu-latest
|
||||
needs: [docker-build, integration-tests]
|
||||
if: github.ref == 'refs/heads/develop' && github.event_name == 'push'
|
||||
environment:
|
||||
name: staging
|
||||
url: https://staging.breakpilot.app
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Deploy to staging server
|
||||
env:
|
||||
STAGING_HOST: ${{ secrets.STAGING_HOST }}
|
||||
STAGING_USER: ${{ secrets.STAGING_USER }}
|
||||
STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
|
||||
run: |
|
||||
# This is a placeholder for actual deployment
|
||||
# Configure based on your staging infrastructure
|
||||
echo "Deploying to staging environment..."
|
||||
echo "Images to deploy:"
|
||||
echo " - ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-consent-service:develop"
|
||||
echo " - ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-backend:develop"
|
||||
echo " - ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-website:develop"
|
||||
|
||||
# Example: SSH deployment (uncomment when configured)
|
||||
# mkdir -p ~/.ssh
|
||||
# echo "$STAGING_SSH_KEY" > ~/.ssh/id_rsa
|
||||
# chmod 600 ~/.ssh/id_rsa
|
||||
# ssh -o StrictHostKeyChecking=no $STAGING_USER@$STAGING_HOST "cd /opt/breakpilot && docker compose pull && docker compose up -d"
|
||||
|
||||
- name: Notify deployment
|
||||
run: |
|
||||
echo "## Staging Deployment" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Successfully deployed to staging environment" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "**Deployed images:**" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- consent-service: \`develop\`" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- backend: \`develop\`" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- website: \`develop\`" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
# ==========================================
|
||||
# Deploy to Production
|
||||
# ==========================================
|
||||
deploy-production:
|
||||
name: Deploy to Production
|
||||
runs-on: ubuntu-latest
|
||||
needs: [docker-build, integration-tests]
|
||||
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
||||
environment:
|
||||
name: production
|
||||
url: https://breakpilot.app
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Deploy to production server
|
||||
env:
|
||||
PROD_HOST: ${{ secrets.PROD_HOST }}
|
||||
PROD_USER: ${{ secrets.PROD_USER }}
|
||||
PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
|
||||
run: |
|
||||
# This is a placeholder for actual deployment
|
||||
# Configure based on your production infrastructure
|
||||
echo "Deploying to production environment..."
|
||||
echo "Images to deploy:"
|
||||
echo " - ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-consent-service:latest"
|
||||
echo " - ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-backend:latest"
|
||||
echo " - ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-website:latest"
|
||||
|
||||
# Example: SSH deployment (uncomment when configured)
|
||||
# mkdir -p ~/.ssh
|
||||
# echo "$PROD_SSH_KEY" > ~/.ssh/id_rsa
|
||||
# chmod 600 ~/.ssh/id_rsa
|
||||
# ssh -o StrictHostKeyChecking=no $PROD_USER@$PROD_HOST "cd /opt/breakpilot && docker compose pull && docker compose up -d"
|
||||
|
||||
- name: Notify deployment
|
||||
run: |
|
||||
echo "## Production Deployment" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Successfully deployed to production environment" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "**Deployed images:**" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- consent-service: \`latest\`" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- backend: \`latest\`" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- website: \`latest\`" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
# ==========================================
|
||||
# Summary
|
||||
# ==========================================
|
||||
summary:
|
||||
name: CI Summary
|
||||
runs-on: ubuntu-latest
|
||||
needs: [go-tests, python-tests, website-tests, lint, security, docker-build, integration-tests]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Check job results
|
||||
run: |
|
||||
echo "## CI/CD Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Go Tests | ${{ needs.go-tests.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Python Tests | ${{ needs.python-tests.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Website Tests | ${{ needs.website-tests.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Linting | ${{ needs.lint.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Security | ${{ needs.security.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Docker Build | ${{ needs.docker-build.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Integration Tests | ${{ needs.integration-tests.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Docker Images" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Images are pushed to: \`${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}-*\`" >> $GITHUB_STEP_SUMMARY
|
||||
222
admin-v2/.github/workflows/security.yml
vendored
Normal file
222
admin-v2/.github/workflows/security.yml
vendored
Normal file
@@ -0,0 +1,222 @@
|
||||
name: Security Scanning
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, develop]
|
||||
pull_request:
|
||||
branches: [main, develop]
|
||||
schedule:
|
||||
# Run security scans weekly on Sundays at midnight
|
||||
- cron: '0 0 * * 0'
|
||||
|
||||
jobs:
|
||||
# ==========================================
|
||||
# Secret Scanning
|
||||
# ==========================================
|
||||
secret-scan:
|
||||
name: Secret Scanning
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: TruffleHog Secret Scan
|
||||
uses: trufflesecurity/trufflehog@main
|
||||
with:
|
||||
extra_args: --only-verified
|
||||
|
||||
- name: GitLeaks Secret Scan
|
||||
uses: gitleaks/gitleaks-action@v2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Dependency Vulnerability Scanning
|
||||
# ==========================================
|
||||
dependency-scan:
|
||||
name: Dependency Vulnerability Scan
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run Trivy vulnerability scanner (filesystem)
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
scan-type: 'fs'
|
||||
scan-ref: '.'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
format: 'sarif'
|
||||
output: 'trivy-fs-results.sarif'
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload Trivy scan results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
with:
|
||||
sarif_file: 'trivy-fs-results.sarif'
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Go Security Scan
|
||||
# ==========================================
|
||||
go-security:
|
||||
name: Go Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Run Gosec Security Scanner
|
||||
uses: securego/gosec@master
|
||||
with:
|
||||
args: '-no-fail -fmt sarif -out gosec-results.sarif ./consent-service/...'
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload Gosec results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
with:
|
||||
sarif_file: 'gosec-results.sarif'
|
||||
continue-on-error: true
|
||||
|
||||
- name: Run govulncheck
|
||||
working-directory: ./consent-service
|
||||
run: |
|
||||
go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
govulncheck ./... || true
|
||||
|
||||
# ==========================================
|
||||
# Python Security Scan
|
||||
# ==========================================
|
||||
python-security:
|
||||
name: Python Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install safety
|
||||
run: pip install safety bandit
|
||||
|
||||
- name: Run Safety (dependency check)
|
||||
working-directory: ./backend
|
||||
run: safety check -r requirements.txt --full-report || true
|
||||
|
||||
- name: Run Bandit (code security scan)
|
||||
working-directory: ./backend
|
||||
run: bandit -r . -f sarif -o bandit-results.sarif --exit-zero
|
||||
|
||||
- name: Upload Bandit results to GitHub Security tab
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
with:
|
||||
sarif_file: './backend/bandit-results.sarif'
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Node.js Security Scan
|
||||
# ==========================================
|
||||
node-security:
|
||||
name: Node.js Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '20'
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ./website
|
||||
run: npm ci
|
||||
|
||||
- name: Run npm audit
|
||||
working-directory: ./website
|
||||
run: npm audit --audit-level=high || true
|
||||
|
||||
# ==========================================
|
||||
# Docker Image Scanning
|
||||
# ==========================================
|
||||
docker-security:
|
||||
name: Docker Image Security
|
||||
runs-on: ubuntu-latest
|
||||
needs: [go-security, python-security, node-security]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build consent-service image
|
||||
run: docker build -t breakpilot/consent-service:scan ./consent-service
|
||||
|
||||
- name: Run Trivy on consent-service
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: 'breakpilot/consent-service:scan'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
format: 'sarif'
|
||||
output: 'trivy-consent-results.sarif'
|
||||
continue-on-error: true
|
||||
|
||||
- name: Build backend image
|
||||
run: docker build -t breakpilot/backend:scan ./backend
|
||||
|
||||
- name: Run Trivy on backend
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
image-ref: 'breakpilot/backend:scan'
|
||||
severity: 'CRITICAL,HIGH'
|
||||
format: 'sarif'
|
||||
output: 'trivy-backend-results.sarif'
|
||||
continue-on-error: true
|
||||
|
||||
- name: Upload Trivy results
|
||||
uses: github/codeql-action/upload-sarif@v3
|
||||
with:
|
||||
sarif_file: 'trivy-consent-results.sarif'
|
||||
continue-on-error: true
|
||||
|
||||
# ==========================================
|
||||
# Security Summary
|
||||
# ==========================================
|
||||
security-summary:
|
||||
name: Security Summary
|
||||
runs-on: ubuntu-latest
|
||||
needs: [secret-scan, dependency-scan, go-security, python-security, node-security, docker-security]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Create security summary
|
||||
run: |
|
||||
echo "## Security Scan Results" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Scan Type | Status |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "|-----------|--------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Secret Scanning | ${{ needs.secret-scan.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Dependency Scanning | ${{ needs.dependency-scan.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Go Security | ${{ needs.go-security.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Python Security | ${{ needs.python-security.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Node.js Security | ${{ needs.node-security.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| Docker Security | ${{ needs.docker-security.result }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Notes" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- Results are uploaded to the GitHub Security tab" >> $GITHUB_STEP_SUMMARY
|
||||
echo "- Weekly scheduled scans run on Sundays" >> $GITHUB_STEP_SUMMARY
|
||||
244
admin-v2/.github/workflows/test.yml
vendored
Normal file
244
admin-v2/.github/workflows/test.yml
vendored
Normal file
@@ -0,0 +1,244 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ main, develop ]
|
||||
pull_request:
|
||||
branches: [ main, develop ]
|
||||
|
||||
jobs:
|
||||
go-tests:
|
||||
name: Go Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: breakpilot
|
||||
POSTGRES_PASSWORD: breakpilot123
|
||||
POSTGRES_DB: breakpilot_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.21'
|
||||
cache: true
|
||||
cache-dependency-path: consent-service/go.sum
|
||||
|
||||
- name: Install Dependencies
|
||||
working-directory: ./consent-service
|
||||
run: go mod download
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: ./consent-service
|
||||
env:
|
||||
DATABASE_URL: postgres://breakpilot:breakpilot123@localhost:5432/breakpilot_test?sslmode=disable
|
||||
JWT_SECRET: test-secret-key-for-ci
|
||||
JWT_REFRESH_SECRET: test-refresh-secret-for-ci
|
||||
run: |
|
||||
go test -v -race -coverprofile=coverage.out ./...
|
||||
go tool cover -func=coverage.out
|
||||
|
||||
- name: Check Coverage Threshold
|
||||
working-directory: ./consent-service
|
||||
run: |
|
||||
COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total Coverage: $COVERAGE%"
|
||||
if (( $(echo "$COVERAGE < 70.0" | bc -l) )); then
|
||||
echo "Coverage $COVERAGE% is below threshold 70%"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
files: ./consent-service/coverage.out
|
||||
flags: go
|
||||
name: go-coverage
|
||||
|
||||
python-tests:
|
||||
name: Python Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
cache: 'pip'
|
||||
cache-dependency-path: backend/requirements.txt
|
||||
|
||||
- name: Install Dependencies
|
||||
working-directory: ./backend
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
pip install pytest pytest-cov pytest-asyncio
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: ./backend
|
||||
env:
|
||||
CONSENT_SERVICE_URL: http://localhost:8081
|
||||
JWT_SECRET: test-secret-key-for-ci
|
||||
run: |
|
||||
pytest -v --cov=. --cov-report=xml --cov-report=term
|
||||
|
||||
- name: Check Coverage Threshold
|
||||
working-directory: ./backend
|
||||
run: |
|
||||
COVERAGE=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); print(tree.getroot().attrib['line-rate'])")
|
||||
COVERAGE_PCT=$(echo "$COVERAGE * 100" | bc)
|
||||
echo "Total Coverage: ${COVERAGE_PCT}%"
|
||||
if (( $(echo "$COVERAGE_PCT < 60.0" | bc -l) )); then
|
||||
echo "Coverage ${COVERAGE_PCT}% is below threshold 60%"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
files: ./backend/coverage.xml
|
||||
flags: python
|
||||
name: python-coverage
|
||||
|
||||
integration-tests:
|
||||
name: Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Start Services
|
||||
run: |
|
||||
docker-compose up -d
|
||||
docker-compose ps
|
||||
|
||||
- name: Wait for Postgres
|
||||
run: |
|
||||
timeout 60 bash -c 'until docker-compose exec -T postgres pg_isready -U breakpilot; do sleep 2; done'
|
||||
|
||||
- name: Wait for Consent Service
|
||||
run: |
|
||||
timeout 60 bash -c 'until curl -f http://localhost:8081/health; do sleep 2; done'
|
||||
|
||||
- name: Wait for Backend
|
||||
run: |
|
||||
timeout 60 bash -c 'until curl -f http://localhost:8000/health; do sleep 2; done'
|
||||
|
||||
- name: Wait for Mailpit
|
||||
run: |
|
||||
timeout 60 bash -c 'until curl -f http://localhost:8025/api/v1/info; do sleep 2; done'
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
chmod +x ./scripts/integration-tests.sh
|
||||
./scripts/integration-tests.sh
|
||||
|
||||
- name: Show Service Logs on Failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Consent Service Logs ==="
|
||||
docker-compose logs consent-service
|
||||
echo "=== Backend Logs ==="
|
||||
docker-compose logs backend
|
||||
echo "=== Postgres Logs ==="
|
||||
docker-compose logs postgres
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: docker-compose down -v
|
||||
|
||||
lint-go:
|
||||
name: Go Lint
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
version: latest
|
||||
working-directory: consent-service
|
||||
args: --timeout=5m
|
||||
|
||||
lint-python:
|
||||
name: Python Lint
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
pip install flake8 black mypy
|
||||
|
||||
- name: Run Black
|
||||
working-directory: ./backend
|
||||
run: black --check .
|
||||
|
||||
- name: Run Flake8
|
||||
working-directory: ./backend
|
||||
run: flake8 . --max-line-length=120 --exclude=venv
|
||||
|
||||
security-scan:
|
||||
name: Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Run Trivy Security Scan
|
||||
uses: aquasecurity/trivy-action@master
|
||||
with:
|
||||
scan-type: 'fs'
|
||||
scan-ref: '.'
|
||||
format: 'sarif'
|
||||
output: 'trivy-results.sarif'
|
||||
|
||||
- name: Upload Trivy Results to GitHub Security
|
||||
uses: github/codeql-action/upload-sarif@v2
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: 'trivy-results.sarif'
|
||||
|
||||
all-checks:
|
||||
name: All Checks Passed
|
||||
runs-on: ubuntu-latest
|
||||
needs: [go-tests, python-tests, integration-tests, lint-go, lint-python, security-scan]
|
||||
|
||||
steps:
|
||||
- name: All Tests Passed
|
||||
run: echo "All tests and checks passed successfully!"
|
||||
167
admin-v2/.gitignore
vendored
Normal file
167
admin-v2/.gitignore
vendored
Normal file
@@ -0,0 +1,167 @@
|
||||
# ============================================
|
||||
# BreakPilot PWA - Git Ignore
|
||||
# ============================================
|
||||
|
||||
# Environment files (keep examples only)
|
||||
.env
|
||||
.env.local
|
||||
*.env.local
|
||||
|
||||
# Keep examples and environment templates
|
||||
!.env.example
|
||||
!.env.dev
|
||||
!.env.staging
|
||||
# .env.prod should NOT be in repo (contains production secrets)
|
||||
|
||||
# ============================================
|
||||
# Python
|
||||
# ============================================
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
venv/
|
||||
ENV/
|
||||
.venv/
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
*.egg
|
||||
.pytest_cache/
|
||||
htmlcov/
|
||||
.coverage
|
||||
.coverage.*
|
||||
coverage.xml
|
||||
*.cover
|
||||
|
||||
# ============================================
|
||||
# Node.js
|
||||
# ============================================
|
||||
node_modules/
|
||||
.next/
|
||||
out/
|
||||
dist/
|
||||
build/
|
||||
.npm
|
||||
.yarn-integrity
|
||||
*.tsbuildinfo
|
||||
|
||||
# ============================================
|
||||
# Go
|
||||
# ============================================
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.dylib
|
||||
*.test
|
||||
*.out
|
||||
vendor/
|
||||
|
||||
# ============================================
|
||||
# Docker
|
||||
# ============================================
|
||||
# Don't ignore docker-compose files
|
||||
# Ignore volume data if mounted locally
|
||||
backups/
|
||||
*.sql.gz
|
||||
*.sql
|
||||
|
||||
# ============================================
|
||||
# IDE & Editors
|
||||
# ============================================
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.project
|
||||
.classpath
|
||||
.settings/
|
||||
*.sublime-workspace
|
||||
*.sublime-project
|
||||
|
||||
# ============================================
|
||||
# OS Files
|
||||
# ============================================
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
|
||||
# ============================================
|
||||
# Secrets & Credentials
|
||||
# ============================================
|
||||
secrets/
|
||||
*.pem
|
||||
*.key
|
||||
*.crt
|
||||
*.p12
|
||||
*.pfx
|
||||
credentials.json
|
||||
service-account.json
|
||||
|
||||
# ============================================
|
||||
# Logs
|
||||
# ============================================
|
||||
*.log
|
||||
logs/
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# ============================================
|
||||
# Build Artifacts
|
||||
# ============================================
|
||||
*.zip
|
||||
*.tar.gz
|
||||
*.rar
|
||||
|
||||
# ============================================
|
||||
# Temporary Files
|
||||
# ============================================
|
||||
tmp/
|
||||
temp/
|
||||
*.tmp
|
||||
*.temp
|
||||
|
||||
# ============================================
|
||||
# Test Results
|
||||
# ============================================
|
||||
test-results/
|
||||
playwright-report/
|
||||
coverage/
|
||||
|
||||
# ============================================
|
||||
# ML Models (large files)
|
||||
# ============================================
|
||||
*.pt
|
||||
*.pth
|
||||
*.onnx
|
||||
*.safetensors
|
||||
models/
|
||||
.claude/settings.local.json
|
||||
|
||||
# ============================================
|
||||
# IDE Plugins & AI Tools
|
||||
# ============================================
|
||||
.continue/
|
||||
CLAUDE_CONTINUE.md
|
||||
|
||||
# ============================================
|
||||
# Misplaced / Large Directories
|
||||
# ============================================
|
||||
backend/BreakpilotDrive/
|
||||
backend/website/
|
||||
backend/screenshots/
|
||||
**/za-download-9/
|
||||
|
||||
# ============================================
|
||||
# Debug & Temp Artifacts
|
||||
# ============================================
|
||||
*.command
|
||||
ssh_key*.txt
|
||||
anleitung.txt
|
||||
fix_permissions.txt
|
||||
77
admin-v2/.gitleaks.toml
Normal file
77
admin-v2/.gitleaks.toml
Normal file
@@ -0,0 +1,77 @@
|
||||
# Gitleaks Configuration for BreakPilot
|
||||
# https://github.com/gitleaks/gitleaks
|
||||
#
|
||||
# Run locally: gitleaks detect --source . -v
|
||||
# Pre-commit: gitleaks protect --staged -v
|
||||
|
||||
title = "BreakPilot Gitleaks Configuration"
|
||||
|
||||
# Use the default rules plus custom rules
|
||||
[extend]
|
||||
useDefault = true
|
||||
|
||||
# Custom rules for BreakPilot-specific patterns
|
||||
[[rules]]
|
||||
id = "anthropic-api-key"
|
||||
description = "Anthropic API Key"
|
||||
regex = '''sk-ant-api[0-9a-zA-Z-_]{20,}'''
|
||||
tags = ["api", "anthropic"]
|
||||
keywords = ["sk-ant-api"]
|
||||
|
||||
[[rules]]
|
||||
id = "vast-api-key"
|
||||
description = "vast.ai API Key"
|
||||
regex = '''(?i)(vast[_-]?api[_-]?key|vast[_-]?key)\s*[=:]\s*['"]?([a-zA-Z0-9-_]{20,})['"]?'''
|
||||
tags = ["api", "vast"]
|
||||
keywords = ["vast"]
|
||||
|
||||
[[rules]]
|
||||
id = "stripe-secret-key"
|
||||
description = "Stripe Secret Key"
|
||||
regex = '''sk_live_[0-9a-zA-Z]{24,}'''
|
||||
tags = ["api", "stripe"]
|
||||
keywords = ["sk_live"]
|
||||
|
||||
[[rules]]
|
||||
id = "stripe-restricted-key"
|
||||
description = "Stripe Restricted Key"
|
||||
regex = '''rk_live_[0-9a-zA-Z]{24,}'''
|
||||
tags = ["api", "stripe"]
|
||||
keywords = ["rk_live"]
|
||||
|
||||
[[rules]]
|
||||
id = "jwt-secret-hardcoded"
|
||||
description = "Hardcoded JWT Secret"
|
||||
regex = '''(?i)(jwt[_-]?secret|jwt[_-]?key)\s*[=:]\s*['"]([^'"]{32,})['"]'''
|
||||
tags = ["secret", "jwt"]
|
||||
keywords = ["jwt"]
|
||||
|
||||
# Allowlist for false positives
|
||||
[allowlist]
|
||||
description = "Global allowlist"
|
||||
paths = [
|
||||
'''\.env\.example$''',
|
||||
'''\.env\.template$''',
|
||||
'''docs/.*\.md$''',
|
||||
'''SBOM\.md$''',
|
||||
'''.*_test\.py$''',
|
||||
'''.*_test\.go$''',
|
||||
'''test_.*\.py$''',
|
||||
'''.*\.bak$''',
|
||||
'''node_modules/.*''',
|
||||
'''venv/.*''',
|
||||
'''\.git/.*''',
|
||||
]
|
||||
|
||||
# Specific commit allowlist (for already-rotated secrets)
|
||||
commits = []
|
||||
|
||||
# Regex patterns to ignore
|
||||
regexes = [
|
||||
'''REPLACE_WITH_REAL_.*''',
|
||||
'''your-.*-key-change-in-production''',
|
||||
'''breakpilot-dev-.*''',
|
||||
'''DEVELOPMENT-ONLY-.*''',
|
||||
'''placeholder.*''',
|
||||
'''example.*key''',
|
||||
]
|
||||
152
admin-v2/.pre-commit-config.yaml
Normal file
152
admin-v2/.pre-commit-config.yaml
Normal file
@@ -0,0 +1,152 @@
|
||||
# Pre-commit Hooks für BreakPilot
|
||||
# Installation: pip install pre-commit && pre-commit install
|
||||
# Aktivierung: pre-commit install
|
||||
|
||||
repos:
|
||||
# Go Hooks
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: go-test
|
||||
name: Go Tests
|
||||
entry: bash -c 'cd consent-service && go test -short ./...'
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: \.go$
|
||||
stages: [commit]
|
||||
|
||||
- id: go-fmt
|
||||
name: Go Format
|
||||
entry: bash -c 'cd consent-service && gofmt -l -w .'
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: \.go$
|
||||
stages: [commit]
|
||||
|
||||
- id: go-vet
|
||||
name: Go Vet
|
||||
entry: bash -c 'cd consent-service && go vet ./...'
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: \.go$
|
||||
stages: [commit]
|
||||
|
||||
- id: golangci-lint
|
||||
name: Go Lint (golangci-lint)
|
||||
entry: bash -c 'cd consent-service && golangci-lint run --timeout=5m'
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: \.go$
|
||||
stages: [commit]
|
||||
|
||||
# Python Hooks
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: pytest
|
||||
name: Python Tests
|
||||
entry: bash -c 'cd backend && pytest -x'
|
||||
language: system
|
||||
pass_filenames: false
|
||||
files: \.py$
|
||||
stages: [commit]
|
||||
|
||||
- id: black
|
||||
name: Black Format
|
||||
entry: black
|
||||
language: python
|
||||
types: [python]
|
||||
args: [--line-length=120]
|
||||
stages: [commit]
|
||||
|
||||
- id: flake8
|
||||
name: Flake8 Lint
|
||||
entry: flake8
|
||||
language: python
|
||||
types: [python]
|
||||
args: [--max-line-length=120, --exclude=venv]
|
||||
stages: [commit]
|
||||
|
||||
# General Hooks
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.5.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
name: Trim Trailing Whitespace
|
||||
- id: end-of-file-fixer
|
||||
name: Fix End of Files
|
||||
- id: check-yaml
|
||||
name: Check YAML
|
||||
args: [--allow-multiple-documents]
|
||||
- id: check-json
|
||||
name: Check JSON
|
||||
- id: check-added-large-files
|
||||
name: Check Large Files
|
||||
args: [--maxkb=500]
|
||||
- id: detect-private-key
|
||||
name: Detect Private Keys
|
||||
- id: mixed-line-ending
|
||||
name: Fix Mixed Line Endings
|
||||
|
||||
# Security Checks
|
||||
- repo: https://github.com/Yelp/detect-secrets
|
||||
rev: v1.4.0
|
||||
hooks:
|
||||
- id: detect-secrets
|
||||
name: Detect Secrets
|
||||
args: ['--baseline', '.secrets.baseline']
|
||||
exclude: |
|
||||
(?x)^(
|
||||
.*\.lock|
|
||||
.*\.sum|
|
||||
package-lock\.json
|
||||
)$
|
||||
|
||||
# =============================================
|
||||
# DevSecOps: Gitleaks (Secrets Detection)
|
||||
# =============================================
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
rev: v8.18.1
|
||||
hooks:
|
||||
- id: gitleaks
|
||||
name: Gitleaks (secrets detection)
|
||||
entry: gitleaks protect --staged -v --config .gitleaks.toml
|
||||
language: golang
|
||||
pass_filenames: false
|
||||
|
||||
# =============================================
|
||||
# DevSecOps: Semgrep (SAST)
|
||||
# =============================================
|
||||
- repo: https://github.com/returntocorp/semgrep
|
||||
rev: v1.52.0
|
||||
hooks:
|
||||
- id: semgrep
|
||||
name: Semgrep (SAST)
|
||||
args:
|
||||
- --config=auto
|
||||
- --config=.semgrep.yml
|
||||
- --severity=ERROR
|
||||
types_or: [python, javascript, typescript, go]
|
||||
stages: [commit]
|
||||
|
||||
# =============================================
|
||||
# DevSecOps: Bandit (Python Security)
|
||||
# =============================================
|
||||
- repo: https://github.com/PyCQA/bandit
|
||||
rev: 1.7.6
|
||||
hooks:
|
||||
- id: bandit
|
||||
name: Bandit (Python security)
|
||||
args: ["-r", "backend/", "-ll", "-x", "backend/tests/*"]
|
||||
files: ^backend/.*\.py$
|
||||
stages: [commit]
|
||||
|
||||
# Branch Protection
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.5.0
|
||||
hooks:
|
||||
- id: no-commit-to-branch
|
||||
name: Protect main/develop branches
|
||||
args: ['--branch', 'main', '--branch', 'develop']
|
||||
|
||||
# Configuration
|
||||
default_stages: [commit]
|
||||
fail_fast: false
|
||||
147
admin-v2/.semgrep.yml
Normal file
147
admin-v2/.semgrep.yml
Normal file
@@ -0,0 +1,147 @@
|
||||
# Semgrep Configuration for BreakPilot
|
||||
# https://semgrep.dev/
|
||||
#
|
||||
# Run locally: semgrep scan --config auto
|
||||
# Run with this config: semgrep scan --config .semgrep.yml
|
||||
|
||||
rules:
|
||||
# =============================================
|
||||
# Python/FastAPI Security Rules
|
||||
# =============================================
|
||||
|
||||
- id: hardcoded-secret-in-string
|
||||
patterns:
|
||||
- pattern-either:
|
||||
- pattern: |
|
||||
$VAR = "...$SECRET..."
|
||||
- pattern: |
|
||||
$VAR = '...$SECRET...'
|
||||
message: "Potential hardcoded secret detected. Use environment variables or Vault."
|
||||
languages: [python]
|
||||
severity: WARNING
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-798: Use of Hard-coded Credentials"
|
||||
|
||||
- id: sql-injection-fastapi
|
||||
patterns:
|
||||
- pattern-either:
|
||||
- pattern: |
|
||||
$CURSOR.execute(f"...{$USER_INPUT}...")
|
||||
- pattern: |
|
||||
$CURSOR.execute("..." + $USER_INPUT + "...")
|
||||
- pattern: |
|
||||
$CURSOR.execute("..." % $USER_INPUT)
|
||||
message: "Potential SQL injection. Use parameterized queries."
|
||||
languages: [python]
|
||||
severity: ERROR
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-89: SQL Injection"
|
||||
owasp: "A03:2021 - Injection"
|
||||
|
||||
- id: command-injection
|
||||
patterns:
|
||||
- pattern-either:
|
||||
- pattern: os.system($USER_INPUT)
|
||||
- pattern: subprocess.call($USER_INPUT, shell=True)
|
||||
- pattern: subprocess.run($USER_INPUT, shell=True)
|
||||
- pattern: subprocess.Popen($USER_INPUT, shell=True)
|
||||
message: "Potential command injection. Avoid shell=True with user input."
|
||||
languages: [python]
|
||||
severity: ERROR
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-78: OS Command Injection"
|
||||
owasp: "A03:2021 - Injection"
|
||||
|
||||
- id: insecure-jwt-algorithm
|
||||
patterns:
|
||||
- pattern: jwt.decode(..., algorithms=["none"], ...)
|
||||
- pattern: jwt.decode(..., algorithms=["HS256"], verify=False, ...)
|
||||
message: "Insecure JWT algorithm or verification disabled."
|
||||
languages: [python]
|
||||
severity: ERROR
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-347: Improper Verification of Cryptographic Signature"
|
||||
|
||||
- id: path-traversal
|
||||
patterns:
|
||||
- pattern: open(... + $USER_INPUT + ...)
|
||||
- pattern: open(f"...{$USER_INPUT}...")
|
||||
- pattern: Path(...) / $USER_INPUT
|
||||
message: "Potential path traversal. Validate and sanitize file paths."
|
||||
languages: [python]
|
||||
severity: WARNING
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-22: Path Traversal"
|
||||
|
||||
- id: insecure-pickle
|
||||
patterns:
|
||||
- pattern: pickle.loads($DATA)
|
||||
- pattern: pickle.load($FILE)
|
||||
message: "Pickle deserialization is insecure. Use JSON or other safe formats."
|
||||
languages: [python]
|
||||
severity: WARNING
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-502: Deserialization of Untrusted Data"
|
||||
|
||||
# =============================================
|
||||
# Go Security Rules
|
||||
# =============================================
|
||||
|
||||
- id: go-sql-injection
|
||||
patterns:
|
||||
- pattern: |
|
||||
$DB.Query(fmt.Sprintf("...", $USER_INPUT))
|
||||
- pattern: |
|
||||
$DB.Exec(fmt.Sprintf("...", $USER_INPUT))
|
||||
message: "Potential SQL injection in Go. Use parameterized queries."
|
||||
languages: [go]
|
||||
severity: ERROR
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-89: SQL Injection"
|
||||
|
||||
- id: go-hardcoded-credentials
|
||||
patterns:
|
||||
- pattern: |
|
||||
$VAR := "..."
|
||||
- metavariable-regex:
|
||||
metavariable: $VAR
|
||||
regex: (password|secret|apiKey|api_key|token)
|
||||
message: "Potential hardcoded credential. Use environment variables."
|
||||
languages: [go]
|
||||
severity: WARNING
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-798: Use of Hard-coded Credentials"
|
||||
|
||||
# =============================================
|
||||
# JavaScript/TypeScript Security Rules
|
||||
# =============================================
|
||||
|
||||
- id: js-xss-innerhtml
|
||||
patterns:
|
||||
- pattern: $EL.innerHTML = $USER_INPUT
|
||||
message: "Potential XSS via innerHTML. Use textContent or sanitize input."
|
||||
languages: [javascript, typescript]
|
||||
severity: WARNING
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-79: Cross-site Scripting"
|
||||
owasp: "A03:2021 - Injection"
|
||||
|
||||
- id: js-eval
|
||||
patterns:
|
||||
- pattern: eval($CODE)
|
||||
- pattern: new Function($CODE)
|
||||
message: "Avoid eval() and new Function() with dynamic input."
|
||||
languages: [javascript, typescript]
|
||||
severity: ERROR
|
||||
metadata:
|
||||
category: security
|
||||
cwe: "CWE-95: Improper Neutralization of Directives in Dynamically Evaluated Code"
|
||||
66
admin-v2/.trivy.yaml
Normal file
66
admin-v2/.trivy.yaml
Normal file
@@ -0,0 +1,66 @@
|
||||
# Trivy Configuration for BreakPilot
|
||||
# https://trivy.dev/
|
||||
#
|
||||
# Run: trivy image breakpilot-pwa-backend:latest
|
||||
# Run filesystem: trivy fs .
|
||||
# Run config: trivy config .
|
||||
|
||||
# Scan settings
|
||||
scan:
|
||||
# Security checks to perform
|
||||
security-checks:
|
||||
- vuln # Vulnerabilities
|
||||
- config # Misconfigurations
|
||||
- secret # Secrets in files
|
||||
|
||||
# Vulnerability settings
|
||||
vulnerability:
|
||||
# Vulnerability types to scan for
|
||||
type:
|
||||
- os # OS packages
|
||||
- library # Application dependencies
|
||||
|
||||
# Ignore unfixed vulnerabilities
|
||||
ignore-unfixed: false
|
||||
|
||||
# Severity settings
|
||||
severity:
|
||||
- CRITICAL
|
||||
- HIGH
|
||||
- MEDIUM
|
||||
# - LOW # Uncomment to include low severity
|
||||
|
||||
# Output format
|
||||
format: table
|
||||
|
||||
# Exit code on findings
|
||||
exit-code: 1
|
||||
|
||||
# Timeout
|
||||
timeout: 10m
|
||||
|
||||
# Cache directory
|
||||
cache-dir: /tmp/trivy-cache
|
||||
|
||||
# Skip files/directories
|
||||
skip-dirs:
|
||||
- node_modules
|
||||
- venv
|
||||
- .venv
|
||||
- __pycache__
|
||||
- .git
|
||||
- .idea
|
||||
- .vscode
|
||||
|
||||
skip-files:
|
||||
- "*.md"
|
||||
- "*.txt"
|
||||
- "*.log"
|
||||
|
||||
# Ignore specific vulnerabilities (add after review)
|
||||
ignorefile: .trivyignore
|
||||
|
||||
# SBOM generation
|
||||
sbom:
|
||||
format: cyclonedx
|
||||
output: sbom.json
|
||||
9
admin-v2/.trivyignore
Normal file
9
admin-v2/.trivyignore
Normal file
@@ -0,0 +1,9 @@
|
||||
# Trivy Ignore File for BreakPilot
|
||||
# Add vulnerability IDs to ignore after security review
|
||||
# Format: CVE-XXXX-XXXXX or GHSA-xxxx-xxxx-xxxx
|
||||
|
||||
# Example (remove after adding real ignores):
|
||||
# CVE-2021-12345 # Reason: Not exploitable in our context
|
||||
|
||||
# Reviewed and accepted risks:
|
||||
# (Add vulnerabilities here after security team review)
|
||||
132
admin-v2/.woodpecker/auto-fix.yml
Normal file
132
admin-v2/.woodpecker/auto-fix.yml
Normal file
@@ -0,0 +1,132 @@
|
||||
# Woodpecker CI Auto-Fix Pipeline
|
||||
# Automatische Reparatur fehlgeschlagener Tests
|
||||
#
|
||||
# Laeuft taeglich um 2:00 Uhr nachts
|
||||
# Analysiert offene Backlog-Items und versucht automatische Fixes
|
||||
|
||||
when:
|
||||
- event: cron
|
||||
cron: "0 2 * * *" # Taeglich um 2:00 Uhr
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
settings:
|
||||
depth: 1
|
||||
extra_hosts:
|
||||
- macmini:192.168.178.100
|
||||
|
||||
steps:
|
||||
# ========================================
|
||||
# 1. Fetch Failed Tests from Backlog
|
||||
# ========================================
|
||||
|
||||
fetch-backlog:
|
||||
image: curlimages/curl:latest
|
||||
commands:
|
||||
- |
|
||||
curl -s "http://backend:8000/api/tests/backlog?status=open&priority=critical" \
|
||||
-o backlog-critical.json
|
||||
curl -s "http://backend:8000/api/tests/backlog?status=open&priority=high" \
|
||||
-o backlog-high.json
|
||||
- echo "=== Kritische Tests ==="
|
||||
- cat backlog-critical.json | head -50
|
||||
- echo "=== Hohe Prioritaet ==="
|
||||
- cat backlog-high.json | head -50
|
||||
|
||||
# ========================================
|
||||
# 2. Analyze and Classify Errors
|
||||
# ========================================
|
||||
|
||||
analyze-errors:
|
||||
image: python:3.12-slim
|
||||
commands:
|
||||
- pip install --quiet jq-py
|
||||
- |
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
import os
|
||||
|
||||
def classify_error(error_type, error_msg):
|
||||
"""Klassifiziert Fehler nach Auto-Fix-Potential"""
|
||||
auto_fixable = {
|
||||
'nil_pointer': 'high',
|
||||
'import_error': 'high',
|
||||
'undefined_variable': 'medium',
|
||||
'type_error': 'medium',
|
||||
'assertion': 'low',
|
||||
'timeout': 'low',
|
||||
'logic_error': 'manual'
|
||||
}
|
||||
return auto_fixable.get(error_type, 'manual')
|
||||
|
||||
# Lade Backlog
|
||||
try:
|
||||
with open('backlog-critical.json') as f:
|
||||
critical = json.load(f)
|
||||
with open('backlog-high.json') as f:
|
||||
high = json.load(f)
|
||||
except:
|
||||
print("Keine Backlog-Daten gefunden")
|
||||
exit(0)
|
||||
|
||||
all_items = critical.get('items', []) + high.get('items', [])
|
||||
|
||||
auto_fix_candidates = []
|
||||
for item in all_items:
|
||||
fix_potential = classify_error(
|
||||
item.get('error_type', 'unknown'),
|
||||
item.get('error_message', '')
|
||||
)
|
||||
if fix_potential in ['high', 'medium']:
|
||||
auto_fix_candidates.append({
|
||||
'id': item.get('id'),
|
||||
'test_name': item.get('test_name'),
|
||||
'error_type': item.get('error_type'),
|
||||
'fix_potential': fix_potential
|
||||
})
|
||||
|
||||
print(f"Auto-Fix Kandidaten: {len(auto_fix_candidates)}")
|
||||
with open('auto-fix-candidates.json', 'w') as f:
|
||||
json.dump(auto_fix_candidates, f, indent=2)
|
||||
EOF
|
||||
depends_on:
|
||||
- fetch-backlog
|
||||
|
||||
# ========================================
|
||||
# 3. Generate Fix Suggestions (Placeholder)
|
||||
# ========================================
|
||||
|
||||
generate-fixes:
|
||||
image: python:3.12-slim
|
||||
commands:
|
||||
- |
|
||||
echo "Auto-Fix Generation ist in Phase 4 geplant"
|
||||
echo "Aktuell werden nur Vorschlaege generiert"
|
||||
|
||||
# Hier wuerde Claude API oder anderer LLM aufgerufen werden
|
||||
# python3 scripts/auto-fix-agent.py auto-fix-candidates.json
|
||||
|
||||
echo "Fix-Vorschlaege wuerden hier generiert werden"
|
||||
depends_on:
|
||||
- analyze-errors
|
||||
|
||||
# ========================================
|
||||
# 4. Report Results
|
||||
# ========================================
|
||||
|
||||
report-results:
|
||||
image: curlimages/curl:latest
|
||||
commands:
|
||||
- |
|
||||
curl -X POST "http://backend:8000/api/tests/auto-fix/report" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"run_date\": \"$(date -Iseconds)\",
|
||||
\"candidates_found\": $(cat auto-fix-candidates.json | wc -l),
|
||||
\"fixes_attempted\": 0,
|
||||
\"fixes_successful\": 0,
|
||||
\"status\": \"analysis_only\"
|
||||
}" || true
|
||||
when:
|
||||
status: [success, failure]
|
||||
37
admin-v2/.woodpecker/build-ci-image.yml
Normal file
37
admin-v2/.woodpecker/build-ci-image.yml
Normal file
@@ -0,0 +1,37 @@
|
||||
# One-time pipeline to build the custom Python CI image
|
||||
# Trigger manually, then delete this file
|
||||
#
|
||||
# This builds the breakpilot/python-ci:3.12 image on the CI runner
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
settings:
|
||||
depth: 1
|
||||
extra_hosts:
|
||||
- macmini:192.168.178.100
|
||||
|
||||
steps:
|
||||
build-python-ci-image:
|
||||
image: docker:27-cli
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
commands:
|
||||
- |
|
||||
echo "=== Building breakpilot/python-ci:3.12 ==="
|
||||
|
||||
docker build \
|
||||
-t breakpilot/python-ci:3.12 \
|
||||
-t breakpilot/python-ci:latest \
|
||||
-f .docker/python-ci.Dockerfile \
|
||||
.
|
||||
|
||||
echo ""
|
||||
echo "=== Build complete ==="
|
||||
docker images | grep breakpilot/python-ci
|
||||
|
||||
echo ""
|
||||
echo "Image is now available for CI pipelines!"
|
||||
161
admin-v2/.woodpecker/integration.yml
Normal file
161
admin-v2/.woodpecker/integration.yml
Normal file
@@ -0,0 +1,161 @@
|
||||
# Integration Tests Pipeline
|
||||
# Separate Datei weil Services auf Pipeline-Ebene definiert werden muessen
|
||||
#
|
||||
# Diese Pipeline laeuft parallel zur main.yml und testet:
|
||||
# - Database Connectivity (PostgreSQL)
|
||||
# - Cache Connectivity (Valkey/Redis)
|
||||
# - Service-to-Service Kommunikation
|
||||
#
|
||||
# Dokumentation: docs/testing/integration-test-environment.md
|
||||
|
||||
when:
|
||||
- event: [push, pull_request]
|
||||
branch: [main, develop]
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
settings:
|
||||
depth: 1
|
||||
extra_hosts:
|
||||
- macmini:192.168.178.100
|
||||
|
||||
# Services auf Pipeline-Ebene (NICHT Step-Ebene!)
|
||||
# Diese Services sind fuer ALLE Steps verfuegbar
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
environment:
|
||||
POSTGRES_USER: breakpilot
|
||||
POSTGRES_PASSWORD: breakpilot_test
|
||||
POSTGRES_DB: breakpilot_test
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8-alpine
|
||||
|
||||
steps:
|
||||
wait-for-services:
|
||||
image: postgres:16-alpine
|
||||
commands:
|
||||
- |
|
||||
echo "=== Waiting for PostgreSQL ==="
|
||||
for i in $(seq 1 30); do
|
||||
if pg_isready -h postgres -U breakpilot; then
|
||||
echo "PostgreSQL ready after $i attempts!"
|
||||
break
|
||||
fi
|
||||
echo "Attempt $i/30: PostgreSQL not ready, waiting..."
|
||||
sleep 2
|
||||
done
|
||||
# Final check
|
||||
if ! pg_isready -h postgres -U breakpilot; then
|
||||
echo "ERROR: PostgreSQL not ready after 30 attempts"
|
||||
exit 1
|
||||
fi
|
||||
- |
|
||||
echo "=== Waiting for Valkey ==="
|
||||
# Install redis-cli in postgres alpine image
|
||||
apk add --no-cache redis > /dev/null 2>&1 || true
|
||||
for i in $(seq 1 30); do
|
||||
if redis-cli -h valkey ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Valkey ready after $i attempts!"
|
||||
break
|
||||
fi
|
||||
echo "Attempt $i/30: Valkey not ready, waiting..."
|
||||
sleep 2
|
||||
done
|
||||
# Final check
|
||||
if ! redis-cli -h valkey ping 2>/dev/null | grep -q PONG; then
|
||||
echo "ERROR: Valkey not ready after 30 attempts"
|
||||
exit 1
|
||||
fi
|
||||
- echo "=== All services ready ==="
|
||||
|
||||
integration-tests:
|
||||
image: breakpilot/python-ci:3.12
|
||||
environment:
|
||||
CI: "true"
|
||||
DATABASE_URL: postgresql://breakpilot:breakpilot_test@postgres:5432/breakpilot_test
|
||||
VALKEY_URL: redis://valkey:6379
|
||||
REDIS_URL: redis://valkey:6379
|
||||
SKIP_INTEGRATION_TESTS: "false"
|
||||
SKIP_DB_TESTS: "false"
|
||||
SKIP_WEASYPRINT_TESTS: "false"
|
||||
# Test-spezifische Umgebungsvariablen
|
||||
ENVIRONMENT: "testing"
|
||||
JWT_SECRET: "test-secret-key-for-integration-tests"
|
||||
TEACHER_REQUIRE_AUTH: "false"
|
||||
GAME_USE_DATABASE: "false"
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
mkdir -p .ci-results
|
||||
cd backend
|
||||
|
||||
# PYTHONPATH setzen damit lokale Module gefunden werden
|
||||
export PYTHONPATH="$(pwd):${PYTHONPATH:-}"
|
||||
|
||||
echo "=== Installing dependencies ==="
|
||||
pip install --quiet --no-cache-dir -r requirements.txt
|
||||
|
||||
echo "=== Running Integration Tests ==="
|
||||
set +e
|
||||
python -m pytest tests/test_integration/ -v \
|
||||
--tb=short \
|
||||
--json-report \
|
||||
--json-report-file=../.ci-results/test-integration.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
# Ergebnisse auswerten
|
||||
if [ -f ../.ci-results/test-integration.json ]; then
|
||||
TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-integration.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0")
|
||||
PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-integration.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0")
|
||||
FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-integration.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0")
|
||||
SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-integration.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0")
|
||||
else
|
||||
echo "WARNUNG: Keine JSON-Ergebnisse gefunden"
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
echo "{\"service\":\"integration-tests\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-integration.json
|
||||
cat ../.ci-results/results-integration.json
|
||||
|
||||
echo ""
|
||||
echo "=== Integration Test Summary ==="
|
||||
echo "Total: $TOTAL | Passed: $PASSED | Failed: $FAILED | Skipped: $SKIPPED"
|
||||
|
||||
if [ "$TEST_EXIT" -ne "0" ]; then
|
||||
echo "Integration tests failed with exit code $TEST_EXIT"
|
||||
exit 1
|
||||
fi
|
||||
depends_on:
|
||||
- wait-for-services
|
||||
|
||||
report-integration-results:
|
||||
image: curlimages/curl:8.10.1
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
echo "=== Sende Integration Test-Ergebnisse an Dashboard ==="
|
||||
|
||||
if [ -f .ci-results/results-integration.json ]; then
|
||||
echo "Sending integration test results..."
|
||||
curl -f -sS -X POST "http://backend:8000/api/tests/ci-result" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"pipeline_id\": \"${CI_PIPELINE_NUMBER}\",
|
||||
\"commit\": \"${CI_COMMIT_SHA}\",
|
||||
\"branch\": \"${CI_COMMIT_BRANCH}\",
|
||||
\"status\": \"${CI_PIPELINE_STATUS:-unknown}\",
|
||||
\"test_results\": $(cat .ci-results/results-integration.json)
|
||||
}" || echo "WARNUNG: Konnte Ergebnisse nicht an Dashboard senden"
|
||||
else
|
||||
echo "Keine Integration-Ergebnisse zum Senden gefunden"
|
||||
fi
|
||||
|
||||
echo "=== Integration Test-Ergebnisse gesendet ==="
|
||||
when:
|
||||
status: [success, failure]
|
||||
depends_on:
|
||||
- integration-tests
|
||||
669
admin-v2/.woodpecker/main.yml
Normal file
669
admin-v2/.woodpecker/main.yml
Normal file
@@ -0,0 +1,669 @@
|
||||
# Woodpecker CI Main Pipeline
|
||||
# BreakPilot PWA - CI/CD Pipeline
|
||||
#
|
||||
# Plattform: ARM64 (Apple Silicon Mac Mini)
|
||||
#
|
||||
# Strategie:
|
||||
# - Tests laufen bei JEDEM Push/PR
|
||||
# - Test-Ergebnisse werden an Dashboard gesendet
|
||||
# - Builds/Scans laufen nur bei Tags oder manuell
|
||||
# - Deployment nur manuell (Sicherheit)
|
||||
|
||||
when:
|
||||
- event: [push, pull_request, manual, tag]
|
||||
branch: [main, develop]
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
settings:
|
||||
depth: 1
|
||||
extra_hosts:
|
||||
- macmini:192.168.178.100
|
||||
|
||||
variables:
|
||||
- &golang_image golang:1.23-alpine
|
||||
- &python_image python:3.12-slim
|
||||
- &python_ci_image breakpilot/python-ci:3.12 # Custom image with WeasyPrint
|
||||
- &nodejs_image node:20-alpine
|
||||
- &docker_image docker:27-cli
|
||||
|
||||
steps:
|
||||
# ========================================
|
||||
# STAGE 1: Lint (nur bei PRs)
|
||||
# ========================================
|
||||
|
||||
go-lint:
|
||||
image: golangci/golangci-lint:v1.55-alpine
|
||||
commands:
|
||||
- cd consent-service && golangci-lint run --timeout 5m ./...
|
||||
- cd ../billing-service && golangci-lint run --timeout 5m ./...
|
||||
- cd ../school-service && golangci-lint run --timeout 5m ./...
|
||||
when:
|
||||
event: pull_request
|
||||
|
||||
python-lint:
|
||||
image: *python_image
|
||||
commands:
|
||||
- pip install --quiet ruff black
|
||||
- ruff check backend/ --output-format=github || true
|
||||
- black --check backend/ || true
|
||||
when:
|
||||
event: pull_request
|
||||
|
||||
# ========================================
|
||||
# STAGE 2: Unit Tests mit JSON-Ausgabe
|
||||
# Ergebnisse werden im Workspace gespeichert (.ci-results/)
|
||||
# ========================================
|
||||
|
||||
test-go-consent:
|
||||
image: *golang_image
|
||||
environment:
|
||||
CGO_ENABLED: "0"
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
apk add --no-cache jq bash
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "consent-service" ]; then
|
||||
echo '{"service":"consent-service","framework":"go","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-consent.json
|
||||
echo "WARNUNG: consent-service Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd consent-service
|
||||
set +e
|
||||
go test -v -json -coverprofile=coverage.out ./... 2>&1 | tee ../.ci-results/test-consent.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
# JSON-Zeilen extrahieren und mit jq zählen
|
||||
JSON_FILE="../.ci-results/test-consent.json"
|
||||
if grep -q '^{' "$JSON_FILE" 2>/dev/null; then
|
||||
TOTAL=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="run" and .Test != null)] | length')
|
||||
PASSED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="pass" and .Test != null)] | length')
|
||||
FAILED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="fail" and .Test != null)] | length')
|
||||
SKIPPED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="skip" and .Test != null)] | length')
|
||||
else
|
||||
echo "WARNUNG: Keine JSON-Zeilen in $JSON_FILE gefunden (Build-Fehler?)"
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
COVERAGE=$(go tool cover -func=coverage.out 2>/dev/null | tail -1 | awk '{print $3}' | tr -d '%' || echo "0")
|
||||
[ -z "$COVERAGE" ] && COVERAGE=0
|
||||
|
||||
echo "{\"service\":\"consent-service\",\"framework\":\"go\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":$COVERAGE}" > ../.ci-results/results-consent.json
|
||||
cat ../.ci-results/results-consent.json
|
||||
|
||||
# Backlog-Strategie: Fehler werden gemeldet aber Pipeline laeuft weiter
|
||||
if [ "$FAILED" -gt "0" ]; then
|
||||
echo "WARNUNG: $FAILED Tests fehlgeschlagen - werden ins Backlog geschrieben"
|
||||
fi
|
||||
|
||||
test-go-billing:
|
||||
image: *golang_image
|
||||
environment:
|
||||
CGO_ENABLED: "0"
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
apk add --no-cache jq bash
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "billing-service" ]; then
|
||||
echo '{"service":"billing-service","framework":"go","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-billing.json
|
||||
echo "WARNUNG: billing-service Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd billing-service
|
||||
set +e
|
||||
go test -v -json -coverprofile=coverage.out ./... 2>&1 | tee ../.ci-results/test-billing.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
# JSON-Zeilen extrahieren und mit jq zählen
|
||||
JSON_FILE="../.ci-results/test-billing.json"
|
||||
if grep -q '^{' "$JSON_FILE" 2>/dev/null; then
|
||||
TOTAL=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="run" and .Test != null)] | length')
|
||||
PASSED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="pass" and .Test != null)] | length')
|
||||
FAILED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="fail" and .Test != null)] | length')
|
||||
SKIPPED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="skip" and .Test != null)] | length')
|
||||
else
|
||||
echo "WARNUNG: Keine JSON-Zeilen in $JSON_FILE gefunden (Build-Fehler?)"
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
COVERAGE=$(go tool cover -func=coverage.out 2>/dev/null | tail -1 | awk '{print $3}' | tr -d '%' || echo "0")
|
||||
[ -z "$COVERAGE" ] && COVERAGE=0
|
||||
|
||||
echo "{\"service\":\"billing-service\",\"framework\":\"go\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":$COVERAGE}" > ../.ci-results/results-billing.json
|
||||
cat ../.ci-results/results-billing.json
|
||||
|
||||
# Backlog-Strategie: Fehler werden gemeldet aber Pipeline laeuft weiter
|
||||
if [ "$FAILED" -gt "0" ]; then
|
||||
echo "WARNUNG: $FAILED Tests fehlgeschlagen - werden ins Backlog geschrieben"
|
||||
fi
|
||||
|
||||
test-go-school:
|
||||
image: *golang_image
|
||||
environment:
|
||||
CGO_ENABLED: "0"
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
apk add --no-cache jq bash
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "school-service" ]; then
|
||||
echo '{"service":"school-service","framework":"go","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-school.json
|
||||
echo "WARNUNG: school-service Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd school-service
|
||||
set +e
|
||||
go test -v -json -coverprofile=coverage.out ./... 2>&1 | tee ../.ci-results/test-school.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
# JSON-Zeilen extrahieren und mit jq zählen
|
||||
JSON_FILE="../.ci-results/test-school.json"
|
||||
if grep -q '^{' "$JSON_FILE" 2>/dev/null; then
|
||||
TOTAL=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="run" and .Test != null)] | length')
|
||||
PASSED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="pass" and .Test != null)] | length')
|
||||
FAILED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="fail" and .Test != null)] | length')
|
||||
SKIPPED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="skip" and .Test != null)] | length')
|
||||
else
|
||||
echo "WARNUNG: Keine JSON-Zeilen in $JSON_FILE gefunden (Build-Fehler?)"
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
COVERAGE=$(go tool cover -func=coverage.out 2>/dev/null | tail -1 | awk '{print $3}' | tr -d '%' || echo "0")
|
||||
[ -z "$COVERAGE" ] && COVERAGE=0
|
||||
|
||||
echo "{\"service\":\"school-service\",\"framework\":\"go\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":$COVERAGE}" > ../.ci-results/results-school.json
|
||||
cat ../.ci-results/results-school.json
|
||||
|
||||
# Backlog-Strategie: Fehler werden gemeldet aber Pipeline laeuft weiter
|
||||
if [ "$FAILED" -gt "0" ]; then
|
||||
echo "WARNUNG: $FAILED Tests fehlgeschlagen - werden ins Backlog geschrieben"
|
||||
fi
|
||||
|
||||
test-go-edu-search:
|
||||
image: *golang_image
|
||||
environment:
|
||||
CGO_ENABLED: "0"
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
apk add --no-cache jq bash
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "edu-search-service" ]; then
|
||||
echo '{"service":"edu-search-service","framework":"go","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-edu-search.json
|
||||
echo "WARNUNG: edu-search-service Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd edu-search-service
|
||||
set +e
|
||||
go test -v -json -coverprofile=coverage.out ./internal/... 2>&1 | tee ../.ci-results/test-edu-search.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
# JSON-Zeilen extrahieren und mit jq zählen
|
||||
JSON_FILE="../.ci-results/test-edu-search.json"
|
||||
if grep -q '^{' "$JSON_FILE" 2>/dev/null; then
|
||||
TOTAL=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="run" and .Test != null)] | length')
|
||||
PASSED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="pass" and .Test != null)] | length')
|
||||
FAILED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="fail" and .Test != null)] | length')
|
||||
SKIPPED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="skip" and .Test != null)] | length')
|
||||
else
|
||||
echo "WARNUNG: Keine JSON-Zeilen in $JSON_FILE gefunden (Build-Fehler?)"
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
COVERAGE=$(go tool cover -func=coverage.out 2>/dev/null | tail -1 | awk '{print $3}' | tr -d '%' || echo "0")
|
||||
[ -z "$COVERAGE" ] && COVERAGE=0
|
||||
|
||||
echo "{\"service\":\"edu-search-service\",\"framework\":\"go\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":$COVERAGE}" > ../.ci-results/results-edu-search.json
|
||||
cat ../.ci-results/results-edu-search.json
|
||||
|
||||
# Backlog-Strategie: Fehler werden gemeldet aber Pipeline laeuft weiter
|
||||
if [ "$FAILED" -gt "0" ]; then
|
||||
echo "WARNUNG: $FAILED Tests fehlgeschlagen - werden ins Backlog geschrieben"
|
||||
fi
|
||||
|
||||
test-go-ai-compliance:
|
||||
image: *golang_image
|
||||
environment:
|
||||
CGO_ENABLED: "0"
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
apk add --no-cache jq bash
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "ai-compliance-sdk" ]; then
|
||||
echo '{"service":"ai-compliance-sdk","framework":"go","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-ai-compliance.json
|
||||
echo "WARNUNG: ai-compliance-sdk Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd ai-compliance-sdk
|
||||
set +e
|
||||
go test -v -json -coverprofile=coverage.out ./... 2>&1 | tee ../.ci-results/test-ai-compliance.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
# JSON-Zeilen extrahieren und mit jq zählen
|
||||
JSON_FILE="../.ci-results/test-ai-compliance.json"
|
||||
if grep -q '^{' "$JSON_FILE" 2>/dev/null; then
|
||||
TOTAL=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="run" and .Test != null)] | length')
|
||||
PASSED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="pass" and .Test != null)] | length')
|
||||
FAILED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="fail" and .Test != null)] | length')
|
||||
SKIPPED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="skip" and .Test != null)] | length')
|
||||
else
|
||||
echo "WARNUNG: Keine JSON-Zeilen in $JSON_FILE gefunden (Build-Fehler?)"
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
COVERAGE=$(go tool cover -func=coverage.out 2>/dev/null | tail -1 | awk '{print $3}' | tr -d '%' || echo "0")
|
||||
[ -z "$COVERAGE" ] && COVERAGE=0
|
||||
|
||||
echo "{\"service\":\"ai-compliance-sdk\",\"framework\":\"go\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":$COVERAGE}" > ../.ci-results/results-ai-compliance.json
|
||||
cat ../.ci-results/results-ai-compliance.json
|
||||
|
||||
# Backlog-Strategie: Fehler werden gemeldet aber Pipeline laeuft weiter
|
||||
if [ "$FAILED" -gt "0" ]; then
|
||||
echo "WARNUNG: $FAILED Tests fehlgeschlagen - werden ins Backlog geschrieben"
|
||||
fi
|
||||
|
||||
test-python-backend:
|
||||
image: *python_ci_image
|
||||
environment:
|
||||
CI: "true"
|
||||
DATABASE_URL: "postgresql://test:test@localhost:5432/test_db"
|
||||
SKIP_DB_TESTS: "true"
|
||||
SKIP_WEASYPRINT_TESTS: "false"
|
||||
SKIP_INTEGRATION_TESTS: "true"
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "backend" ]; then
|
||||
echo '{"service":"backend","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-backend.json
|
||||
echo "WARNUNG: backend Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd backend
|
||||
# Set PYTHONPATH to current directory (backend) so local packages like classroom_engine, alerts_agent are found
|
||||
# IMPORTANT: Use absolute path and export before pip install to ensure modules are available
|
||||
export PYTHONPATH="$(pwd):${PYTHONPATH:-}"
|
||||
|
||||
# Test tools are pre-installed in breakpilot/python-ci image
|
||||
# Only install project-specific dependencies
|
||||
pip install --quiet --no-cache-dir -r requirements.txt
|
||||
|
||||
# NOTE: PostgreSQL service removed - tests that require DB are skipped via SKIP_DB_TESTS=true
|
||||
# For full integration tests, use: docker compose -f docker-compose.test.yml up -d
|
||||
|
||||
set +e
|
||||
# Use python -m pytest to ensure PYTHONPATH is properly applied before pytest starts
|
||||
python -m pytest tests/ -v --tb=short --cov=. --cov-report=term-missing --json-report --json-report-file=../.ci-results/test-backend.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ -f ../.ci-results/test-backend.json ]; then
|
||||
TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0")
|
||||
PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0")
|
||||
FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0")
|
||||
SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0")
|
||||
else
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
echo "{\"service\":\"backend\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-backend.json
|
||||
cat ../.ci-results/results-backend.json
|
||||
|
||||
if [ "$TEST_EXIT" -ne "0" ]; then exit 1; fi
|
||||
|
||||
test-python-voice:
|
||||
image: *python_image
|
||||
environment:
|
||||
CI: "true"
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "voice-service" ]; then
|
||||
echo '{"service":"voice-service","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-voice.json
|
||||
echo "WARNUNG: voice-service Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd voice-service
|
||||
export PYTHONPATH="$(pwd):${PYTHONPATH:-}"
|
||||
pip install --quiet --no-cache-dir -r requirements.txt
|
||||
pip install --quiet --no-cache-dir pytest-json-report
|
||||
|
||||
set +e
|
||||
python -m pytest tests/ -v --tb=short --json-report --json-report-file=../.ci-results/test-voice.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ -f ../.ci-results/test-voice.json ]; then
|
||||
TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-voice.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0")
|
||||
PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-voice.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0")
|
||||
FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-voice.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0")
|
||||
SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-voice.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0")
|
||||
else
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
echo "{\"service\":\"voice-service\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-voice.json
|
||||
cat ../.ci-results/results-voice.json
|
||||
|
||||
if [ "$TEST_EXIT" -ne "0" ]; then exit 1; fi
|
||||
|
||||
test-bqas-golden:
|
||||
image: *python_image
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "voice-service/tests/bqas" ]; then
|
||||
echo '{"service":"bqas-golden","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-bqas-golden.json
|
||||
echo "WARNUNG: voice-service/tests/bqas Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd voice-service
|
||||
export PYTHONPATH="$(pwd):${PYTHONPATH:-}"
|
||||
pip install --quiet --no-cache-dir -r requirements.txt
|
||||
pip install --quiet --no-cache-dir pytest-json-report pytest-asyncio
|
||||
|
||||
set +e
|
||||
python -m pytest tests/bqas/test_golden.py tests/bqas/test_regression.py tests/bqas/test_synthetic.py -v --tb=short --json-report --json-report-file=../.ci-results/test-bqas-golden.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ -f ../.ci-results/test-bqas-golden.json ]; then
|
||||
TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-golden.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0")
|
||||
PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-golden.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0")
|
||||
FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-golden.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0")
|
||||
SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-golden.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0")
|
||||
else
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
echo "{\"service\":\"bqas-golden\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-bqas-golden.json
|
||||
cat ../.ci-results/results-bqas-golden.json
|
||||
|
||||
# BQAS tests may skip if Ollama not available - don't fail pipeline
|
||||
if [ "$FAILED" -gt "0" ]; then exit 1; fi
|
||||
|
||||
test-bqas-rag:
|
||||
image: *python_image
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "voice-service/tests/bqas" ]; then
|
||||
echo '{"service":"bqas-rag","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-bqas-rag.json
|
||||
echo "WARNUNG: voice-service/tests/bqas Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd voice-service
|
||||
export PYTHONPATH="$(pwd):${PYTHONPATH:-}"
|
||||
pip install --quiet --no-cache-dir -r requirements.txt
|
||||
pip install --quiet --no-cache-dir pytest-json-report pytest-asyncio
|
||||
|
||||
set +e
|
||||
python -m pytest tests/bqas/test_rag.py tests/bqas/test_notifier.py -v --tb=short --json-report --json-report-file=../.ci-results/test-bqas-rag.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ -f ../.ci-results/test-bqas-rag.json ]; then
|
||||
TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-rag.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0")
|
||||
PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-rag.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0")
|
||||
FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-rag.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0")
|
||||
SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-bqas-rag.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0")
|
||||
else
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
echo "{\"service\":\"bqas-rag\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-bqas-rag.json
|
||||
cat ../.ci-results/results-bqas-rag.json
|
||||
|
||||
# BQAS tests may skip if Ollama not available - don't fail pipeline
|
||||
if [ "$FAILED" -gt "0" ]; then exit 1; fi
|
||||
|
||||
test-python-klausur:
|
||||
image: *python_image
|
||||
environment:
|
||||
CI: "true"
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "klausur-service/backend" ]; then
|
||||
echo '{"service":"klausur-service","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-klausur.json
|
||||
echo "WARNUNG: klausur-service/backend Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd klausur-service/backend
|
||||
# Set PYTHONPATH to current directory so local modules like hyde, hybrid_search, etc. are found
|
||||
export PYTHONPATH="$(pwd):${PYTHONPATH:-}"
|
||||
|
||||
pip install --quiet --no-cache-dir -r requirements.txt 2>/dev/null || pip install --quiet --no-cache-dir fastapi uvicorn pytest pytest-asyncio pytest-json-report
|
||||
pip install --quiet --no-cache-dir pytest-json-report
|
||||
|
||||
set +e
|
||||
python -m pytest tests/ -v --tb=short --json-report --json-report-file=../../.ci-results/test-klausur.json
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ -f ../../.ci-results/test-klausur.json ]; then
|
||||
TOTAL=$(python3 -c "import json; d=json.load(open('../../.ci-results/test-klausur.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0")
|
||||
PASSED=$(python3 -c "import json; d=json.load(open('../../.ci-results/test-klausur.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0")
|
||||
FAILED=$(python3 -c "import json; d=json.load(open('../../.ci-results/test-klausur.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0")
|
||||
SKIPPED=$(python3 -c "import json; d=json.load(open('../../.ci-results/test-klausur.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0")
|
||||
else
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
echo "{\"service\":\"klausur-service\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../../.ci-results/results-klausur.json
|
||||
cat ../../.ci-results/results-klausur.json
|
||||
|
||||
if [ "$TEST_EXIT" -ne "0" ]; then exit 1; fi
|
||||
|
||||
test-nodejs-h5p:
|
||||
image: *nodejs_image
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
mkdir -p .ci-results
|
||||
|
||||
if [ ! -d "h5p-service" ]; then
|
||||
echo '{"service":"h5p-service","framework":"jest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-h5p.json
|
||||
echo "WARNUNG: h5p-service Verzeichnis nicht gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cd h5p-service
|
||||
npm ci --silent 2>/dev/null || npm install --silent
|
||||
|
||||
set +e
|
||||
npm run test:ci -- --json --outputFile=../.ci-results/test-h5p.json 2>&1
|
||||
TEST_EXIT=$?
|
||||
set -e
|
||||
|
||||
if [ -f ../.ci-results/test-h5p.json ]; then
|
||||
TOTAL=$(node -e "const d=require('../.ci-results/test-h5p.json'); console.log(d.numTotalTests || 0)" 2>/dev/null || echo "0")
|
||||
PASSED=$(node -e "const d=require('../.ci-results/test-h5p.json'); console.log(d.numPassedTests || 0)" 2>/dev/null || echo "0")
|
||||
FAILED=$(node -e "const d=require('../.ci-results/test-h5p.json'); console.log(d.numFailedTests || 0)" 2>/dev/null || echo "0")
|
||||
SKIPPED=$(node -e "const d=require('../.ci-results/test-h5p.json'); console.log(d.numPendingTests || 0)" 2>/dev/null || echo "0")
|
||||
else
|
||||
TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0
|
||||
fi
|
||||
|
||||
[ -z "$TOTAL" ] && TOTAL=0
|
||||
[ -z "$PASSED" ] && PASSED=0
|
||||
[ -z "$FAILED" ] && FAILED=0
|
||||
[ -z "$SKIPPED" ] && SKIPPED=0
|
||||
|
||||
echo "{\"service\":\"h5p-service\",\"framework\":\"jest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-h5p.json
|
||||
cat ../.ci-results/results-h5p.json
|
||||
|
||||
if [ "$TEST_EXIT" -ne "0" ]; then exit 1; fi
|
||||
|
||||
# ========================================
|
||||
# STAGE 2.5: Integration Tests
|
||||
# ========================================
|
||||
# Integration Tests laufen in separater Pipeline:
|
||||
# .woodpecker/integration.yml
|
||||
# (benötigt Pipeline-Level Services für PostgreSQL und Valkey)
|
||||
|
||||
# ========================================
|
||||
# STAGE 3: Test-Ergebnisse an Dashboard senden
|
||||
# ========================================
|
||||
|
||||
report-test-results:
|
||||
image: curlimages/curl:8.10.1
|
||||
commands:
|
||||
- |
|
||||
set -uo pipefail
|
||||
echo "=== Sende Test-Ergebnisse an Dashboard ==="
|
||||
echo "Pipeline Status: ${CI_PIPELINE_STATUS:-unknown}"
|
||||
ls -la .ci-results/ || echo "Verzeichnis nicht gefunden"
|
||||
|
||||
PIPELINE_STATUS="${CI_PIPELINE_STATUS:-unknown}"
|
||||
|
||||
for f in .ci-results/results-*.json; do
|
||||
[ -f "$f" ] || continue
|
||||
echo "Sending: $f"
|
||||
curl -f -sS -X POST "http://backend:8000/api/tests/ci-result" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"pipeline_id\": \"${CI_PIPELINE_NUMBER}\",
|
||||
\"commit\": \"${CI_COMMIT_SHA}\",
|
||||
\"branch\": \"${CI_COMMIT_BRANCH}\",
|
||||
\"status\": \"${PIPELINE_STATUS}\",
|
||||
\"test_results\": $(cat "$f")
|
||||
}" || echo "WARNUNG: Konnte $f nicht senden"
|
||||
done
|
||||
|
||||
echo "=== Test-Ergebnisse gesendet ==="
|
||||
when:
|
||||
status: [success, failure]
|
||||
depends_on:
|
||||
- test-go-consent
|
||||
- test-go-billing
|
||||
- test-go-school
|
||||
- test-go-edu-search
|
||||
- test-go-ai-compliance
|
||||
- test-python-backend
|
||||
- test-python-voice
|
||||
- test-bqas-golden
|
||||
- test-bqas-rag
|
||||
- test-python-klausur
|
||||
- test-nodejs-h5p
|
||||
|
||||
# ========================================
|
||||
# STAGE 4: Build & Security (nur Tags/manuell)
|
||||
# ========================================
|
||||
|
||||
build-consent-service:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- docker build -t breakpilot/consent-service:${CI_COMMIT_SHA:0:8} ./consent-service
|
||||
- docker tag breakpilot/consent-service:${CI_COMMIT_SHA:0:8} breakpilot/consent-service:latest
|
||||
- echo "Built breakpilot/consent-service:${CI_COMMIT_SHA:0:8}"
|
||||
when:
|
||||
- event: tag
|
||||
- event: manual
|
||||
|
||||
build-backend:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- docker build -t breakpilot/backend:${CI_COMMIT_SHA:0:8} ./backend
|
||||
- docker tag breakpilot/backend:${CI_COMMIT_SHA:0:8} breakpilot/backend:latest
|
||||
- echo "Built breakpilot/backend:${CI_COMMIT_SHA:0:8}"
|
||||
when:
|
||||
- event: tag
|
||||
- event: manual
|
||||
|
||||
build-voice-service:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- |
|
||||
if [ -d ./voice-service ]; then
|
||||
docker build -t breakpilot/voice-service:${CI_COMMIT_SHA:0:8} ./voice-service
|
||||
docker tag breakpilot/voice-service:${CI_COMMIT_SHA:0:8} breakpilot/voice-service:latest
|
||||
echo "Built breakpilot/voice-service:${CI_COMMIT_SHA:0:8}"
|
||||
else
|
||||
echo "voice-service Verzeichnis nicht gefunden - ueberspringe"
|
||||
fi
|
||||
when:
|
||||
- event: tag
|
||||
- event: manual
|
||||
|
||||
generate-sbom:
|
||||
image: *golang_image
|
||||
commands:
|
||||
- |
|
||||
echo "Installing syft for ARM64..."
|
||||
wget -qO- https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
|
||||
syft dir:./consent-service -o cyclonedx-json > sbom-consent.json
|
||||
syft dir:./backend -o cyclonedx-json > sbom-backend.json
|
||||
if [ -d ./voice-service ]; then
|
||||
syft dir:./voice-service -o cyclonedx-json > sbom-voice.json
|
||||
fi
|
||||
echo "SBOMs generated successfully"
|
||||
when:
|
||||
- event: tag
|
||||
- event: manual
|
||||
|
||||
vulnerability-scan:
|
||||
image: *golang_image
|
||||
commands:
|
||||
- |
|
||||
echo "Installing grype for ARM64..."
|
||||
wget -qO- https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
|
||||
grype sbom:sbom-consent.json -o table --fail-on critical || true
|
||||
grype sbom:sbom-backend.json -o table --fail-on critical || true
|
||||
if [ -f sbom-voice.json ]; then
|
||||
grype sbom:sbom-voice.json -o table --fail-on critical || true
|
||||
fi
|
||||
when:
|
||||
- event: tag
|
||||
- event: manual
|
||||
depends_on:
|
||||
- generate-sbom
|
||||
|
||||
# ========================================
|
||||
# STAGE 5: Deploy (nur manuell)
|
||||
# ========================================
|
||||
|
||||
deploy-production:
|
||||
image: *docker_image
|
||||
commands:
|
||||
- echo "Deploying to production..."
|
||||
- docker compose -f docker-compose.yml pull || true
|
||||
- docker compose -f docker-compose.yml up -d --remove-orphans || true
|
||||
when:
|
||||
event: manual
|
||||
depends_on:
|
||||
- build-consent-service
|
||||
- build-backend
|
||||
314
admin-v2/.woodpecker/security.yml
Normal file
314
admin-v2/.woodpecker/security.yml
Normal file
@@ -0,0 +1,314 @@
|
||||
# Woodpecker CI Security Pipeline
|
||||
# Dedizierte Security-Scans fuer DevSecOps
|
||||
#
|
||||
# Laeuft taeglich via Cron und bei jedem PR
|
||||
|
||||
when:
|
||||
- event: cron
|
||||
cron: "0 3 * * *" # Taeglich um 3:00 Uhr
|
||||
- event: pull_request
|
||||
|
||||
clone:
|
||||
git:
|
||||
image: woodpeckerci/plugin-git
|
||||
settings:
|
||||
depth: 1
|
||||
extra_hosts:
|
||||
- macmini:192.168.178.100
|
||||
|
||||
steps:
|
||||
# ========================================
|
||||
# Static Analysis
|
||||
# ========================================
|
||||
|
||||
semgrep-scan:
|
||||
image: returntocorp/semgrep:latest
|
||||
commands:
|
||||
- semgrep scan --config auto --json -o semgrep-results.json . || true
|
||||
- |
|
||||
if [ -f semgrep-results.json ]; then
|
||||
echo "=== Semgrep Findings ==="
|
||||
cat semgrep-results.json | head -100
|
||||
fi
|
||||
when:
|
||||
event: [pull_request, cron]
|
||||
|
||||
bandit-python:
|
||||
image: python:3.12-slim
|
||||
commands:
|
||||
- pip install --quiet bandit
|
||||
- bandit -r backend/ -f json -o bandit-results.json || true
|
||||
- |
|
||||
if [ -f bandit-results.json ]; then
|
||||
echo "=== Bandit Findings ==="
|
||||
cat bandit-results.json | head -50
|
||||
fi
|
||||
when:
|
||||
event: [pull_request, cron]
|
||||
|
||||
gosec-go:
|
||||
image: securego/gosec:latest
|
||||
commands:
|
||||
- gosec -fmt json -out gosec-consent.json ./consent-service/... || true
|
||||
- gosec -fmt json -out gosec-billing.json ./billing-service/... || true
|
||||
- echo "Go Security Scan abgeschlossen"
|
||||
when:
|
||||
event: [pull_request, cron]
|
||||
|
||||
# ========================================
|
||||
# Secrets Detection
|
||||
# ========================================
|
||||
|
||||
gitleaks-scan:
|
||||
image: zricethezav/gitleaks:latest
|
||||
commands:
|
||||
- gitleaks detect --source . --report-format json --report-path gitleaks-report.json || true
|
||||
- |
|
||||
if [ -s gitleaks-report.json ]; then
|
||||
echo "=== WARNUNG: Potentielle Secrets gefunden ==="
|
||||
cat gitleaks-report.json
|
||||
else
|
||||
echo "Keine Secrets gefunden"
|
||||
fi
|
||||
|
||||
trufflehog-scan:
|
||||
image: trufflesecurity/trufflehog:latest
|
||||
commands:
|
||||
- trufflehog filesystem . --json > trufflehog-results.json 2>&1 || true
|
||||
- echo "TruffleHog Scan abgeschlossen"
|
||||
|
||||
# ========================================
|
||||
# Dependency Vulnerabilities
|
||||
# ========================================
|
||||
|
||||
npm-audit:
|
||||
image: node:20-alpine
|
||||
commands:
|
||||
- cd website && npm audit --json > ../npm-audit-website.json || true
|
||||
- cd ../studio-v2 && npm audit --json > ../npm-audit-studio.json || true
|
||||
- cd ../admin-v2 && npm audit --json > ../npm-audit-admin.json || true
|
||||
- echo "NPM Audit abgeschlossen"
|
||||
when:
|
||||
event: [pull_request, cron]
|
||||
|
||||
pip-audit:
|
||||
image: python:3.12-slim
|
||||
commands:
|
||||
- pip install --quiet pip-audit
|
||||
- pip-audit -r backend/requirements.txt --format json -o pip-audit-backend.json || true
|
||||
- pip-audit -r voice-service/requirements.txt --format json -o pip-audit-voice.json || true
|
||||
- echo "Pip Audit abgeschlossen"
|
||||
when:
|
||||
event: [pull_request, cron]
|
||||
|
||||
go-vulncheck:
|
||||
image: golang:1.21-alpine
|
||||
commands:
|
||||
- go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
- cd consent-service && govulncheck ./... || true
|
||||
- cd ../billing-service && govulncheck ./... || true
|
||||
- echo "Go Vulncheck abgeschlossen"
|
||||
when:
|
||||
event: [pull_request, cron]
|
||||
|
||||
# ========================================
|
||||
# Container Security
|
||||
# ========================================
|
||||
|
||||
trivy-filesystem:
|
||||
image: aquasec/trivy:latest
|
||||
commands:
|
||||
- trivy fs --severity HIGH,CRITICAL --format json -o trivy-fs.json . || true
|
||||
- echo "Trivy Filesystem Scan abgeschlossen"
|
||||
when:
|
||||
event: cron
|
||||
|
||||
# ========================================
|
||||
# SBOM Generation (taeglich)
|
||||
# ========================================
|
||||
|
||||
daily-sbom:
|
||||
image: anchore/syft:latest
|
||||
commands:
|
||||
- mkdir -p sbom-reports
|
||||
- syft dir:. -o cyclonedx-json > sbom-reports/sbom-full-$(date +%Y%m%d).json
|
||||
- echo "SBOM generiert"
|
||||
when:
|
||||
event: cron
|
||||
|
||||
# ========================================
|
||||
# AUTO-FIX: Dependency Vulnerabilities
|
||||
# Laeuft nur bei Cron (nightly), nicht bei PRs
|
||||
# ========================================
|
||||
|
||||
auto-fix-npm:
|
||||
image: node:20-alpine
|
||||
commands:
|
||||
- apk add --no-cache git
|
||||
- |
|
||||
echo "=== Auto-Fix: NPM Dependencies ==="
|
||||
FIXES_APPLIED=0
|
||||
|
||||
for dir in website studio-v2 admin-v2 h5p-service; do
|
||||
if [ -d "$dir" ] && [ -f "$dir/package.json" ]; then
|
||||
echo "Pruefe $dir..."
|
||||
cd $dir
|
||||
|
||||
# Speichere Hash vor Fix
|
||||
BEFORE=$(md5sum package-lock.json 2>/dev/null || echo "none")
|
||||
|
||||
# npm audit fix (ohne --force fuer sichere Updates)
|
||||
npm audit fix --package-lock-only 2>/dev/null || true
|
||||
|
||||
# Pruefe ob Aenderungen
|
||||
AFTER=$(md5sum package-lock.json 2>/dev/null || echo "none")
|
||||
if [ "$BEFORE" != "$AFTER" ]; then
|
||||
echo " -> Fixes angewendet in $dir"
|
||||
FIXES_APPLIED=$((FIXES_APPLIED + 1))
|
||||
fi
|
||||
|
||||
cd ..
|
||||
fi
|
||||
done
|
||||
|
||||
echo "NPM Auto-Fix abgeschlossen: $FIXES_APPLIED Projekte aktualisiert"
|
||||
echo "NPM_FIXES=$FIXES_APPLIED" >> /tmp/autofix-results.env
|
||||
when:
|
||||
event: cron
|
||||
|
||||
auto-fix-python:
|
||||
image: python:3.12-slim
|
||||
commands:
|
||||
- apt-get update && apt-get install -y git
|
||||
- pip install --quiet pip-audit
|
||||
- |
|
||||
echo "=== Auto-Fix: Python Dependencies ==="
|
||||
FIXES_APPLIED=0
|
||||
|
||||
for reqfile in backend/requirements.txt voice-service/requirements.txt klausur-service/backend/requirements.txt; do
|
||||
if [ -f "$reqfile" ]; then
|
||||
echo "Pruefe $reqfile..."
|
||||
DIR=$(dirname $reqfile)
|
||||
|
||||
# pip-audit mit --fix (aktualisiert requirements.txt)
|
||||
pip-audit -r $reqfile --fix 2>/dev/null || true
|
||||
|
||||
# Pruefe ob requirements.txt geaendert wurde
|
||||
if git diff --quiet $reqfile 2>/dev/null; then
|
||||
echo " -> Keine Aenderungen in $reqfile"
|
||||
else
|
||||
echo " -> Fixes angewendet in $reqfile"
|
||||
FIXES_APPLIED=$((FIXES_APPLIED + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Python Auto-Fix abgeschlossen: $FIXES_APPLIED Dateien aktualisiert"
|
||||
echo "PYTHON_FIXES=$FIXES_APPLIED" >> /tmp/autofix-results.env
|
||||
when:
|
||||
event: cron
|
||||
|
||||
auto-fix-go:
|
||||
image: golang:1.21-alpine
|
||||
commands:
|
||||
- apk add --no-cache git
|
||||
- |
|
||||
echo "=== Auto-Fix: Go Dependencies ==="
|
||||
FIXES_APPLIED=0
|
||||
|
||||
for dir in consent-service billing-service school-service edu-search ai-compliance-sdk; do
|
||||
if [ -d "$dir" ] && [ -f "$dir/go.mod" ]; then
|
||||
echo "Pruefe $dir..."
|
||||
cd $dir
|
||||
|
||||
# Go mod tidy und update
|
||||
go get -u ./... 2>/dev/null || true
|
||||
go mod tidy 2>/dev/null || true
|
||||
|
||||
# Pruefe ob go.mod/go.sum geaendert wurden
|
||||
if git diff --quiet go.mod go.sum 2>/dev/null; then
|
||||
echo " -> Keine Aenderungen in $dir"
|
||||
else
|
||||
echo " -> Updates angewendet in $dir"
|
||||
FIXES_APPLIED=$((FIXES_APPLIED + 1))
|
||||
fi
|
||||
|
||||
cd ..
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Go Auto-Fix abgeschlossen: $FIXES_APPLIED Module aktualisiert"
|
||||
echo "GO_FIXES=$FIXES_APPLIED" >> /tmp/autofix-results.env
|
||||
when:
|
||||
event: cron
|
||||
|
||||
# ========================================
|
||||
# Commit & Push Auto-Fixes
|
||||
# ========================================
|
||||
|
||||
commit-security-fixes:
|
||||
image: alpine/git:latest
|
||||
commands:
|
||||
- |
|
||||
echo "=== Commit Security Fixes ==="
|
||||
|
||||
# Git konfigurieren
|
||||
git config --global user.email "security-bot@breakpilot.de"
|
||||
git config --global user.name "Security Bot"
|
||||
git config --global --add safe.directory /woodpecker/src
|
||||
|
||||
# Pruefe ob es Aenderungen gibt
|
||||
if git diff --quiet && git diff --cached --quiet; then
|
||||
echo "Keine Security-Fixes zum Committen"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Zeige was geaendert wurde
|
||||
echo "Geaenderte Dateien:"
|
||||
git status --short
|
||||
|
||||
# Stage alle relevanten Dateien
|
||||
git add -A \
|
||||
*/package-lock.json \
|
||||
*/requirements.txt \
|
||||
*/go.mod \
|
||||
*/go.sum \
|
||||
2>/dev/null || true
|
||||
|
||||
# Commit erstellen
|
||||
TIMESTAMP=$(date +%Y-%m-%d)
|
||||
git commit -m "fix(security): auto-fix vulnerable dependencies [$TIMESTAMP]
|
||||
|
||||
Automatische Sicherheitsupdates durch CI/CD Pipeline:
|
||||
- npm audit fix fuer Node.js Projekte
|
||||
- pip-audit --fix fuer Python Projekte
|
||||
- go get -u fuer Go Module
|
||||
|
||||
Co-Authored-By: Security Bot <security-bot@breakpilot.de>" || echo "Nichts zu committen"
|
||||
|
||||
# Push zum Repository
|
||||
git push origin HEAD:main || echo "Push fehlgeschlagen - manueller Review erforderlich"
|
||||
|
||||
echo "Security-Fixes committed und gepusht"
|
||||
when:
|
||||
event: cron
|
||||
status: success
|
||||
|
||||
# ========================================
|
||||
# Report to Dashboard
|
||||
# ========================================
|
||||
|
||||
update-security-dashboard:
|
||||
image: curlimages/curl:latest
|
||||
commands:
|
||||
- |
|
||||
curl -X POST "http://backend:8000/api/security/scan-results" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"scan_type\": \"daily\",
|
||||
\"timestamp\": \"$(date -Iseconds)\",
|
||||
\"tools\": [\"semgrep\", \"bandit\", \"gosec\", \"gitleaks\", \"trivy\"]
|
||||
}" || true
|
||||
when:
|
||||
status: [success, failure]
|
||||
event: cron
|
||||
2029
admin-v2/AI_COMPLIANCE_SDK_IMPLEMENTATION_PLAN.md
Normal file
2029
admin-v2/AI_COMPLIANCE_SDK_IMPLEMENTATION_PLAN.md
Normal file
File diff suppressed because it is too large
Load Diff
566
admin-v2/BREAKPILOT_CONSENT_MANAGEMENT_PLAN.md
Normal file
566
admin-v2/BREAKPILOT_CONSENT_MANAGEMENT_PLAN.md
Normal file
@@ -0,0 +1,566 @@
|
||||
# BreakPilot Consent Management System - Projektplan
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Dieses Dokument beschreibt den Plan zur Entwicklung eines vollständigen Consent Management Systems (CMS) für BreakPilot. Das System wird komplett neu entwickelt und ersetzt das bestehende Policy Vault System, das Bugs enthält und nicht optimal funktioniert.
|
||||
|
||||
---
|
||||
|
||||
## Technologie-Entscheidung: Warum welche Sprache?
|
||||
|
||||
### Backend-Optionen im Vergleich
|
||||
|
||||
| Kriterium | Rust | Go | Python (FastAPI) | TypeScript (NestJS) |
|
||||
|-----------|------|-----|------------------|---------------------|
|
||||
| **Performance** | Exzellent | Sehr gut | Gut | Gut |
|
||||
| **Memory Safety** | Garantiert | GC | GC | GC |
|
||||
| **Entwicklungsgeschwindigkeit** | Langsam | Mittel | Schnell | Schnell |
|
||||
| **Lernkurve** | Steil | Flach | Flach | Mittel |
|
||||
| **Ecosystem für Web** | Wachsend | Sehr gut | Exzellent | Exzellent |
|
||||
| **Integration mit BreakPilot** | Neu | Neu | Bereits vorhanden | Möglich |
|
||||
| **Team-Erfahrung** | ? | ? | Vorhanden | Möglich |
|
||||
|
||||
### Empfehlung: **Python (FastAPI)** oder **Go**
|
||||
|
||||
#### Option A: Python mit FastAPI (Empfohlen für schnelle Integration)
|
||||
**Vorteile:**
|
||||
- Bereits im BreakPilot-Projekt verwendet
|
||||
- Schnelle Entwicklung
|
||||
- Exzellente Dokumentation (automatisch generiert)
|
||||
- Einfache Integration mit bestehendem Code
|
||||
- Type Hints für bessere Code-Qualität
|
||||
- Async/Await Support
|
||||
|
||||
**Nachteile:**
|
||||
- Langsamer als Rust/Go bei hoher Last
|
||||
- GIL-Einschränkungen bei CPU-intensiven Tasks
|
||||
|
||||
#### Option B: Go (Empfohlen für Microservice-Architektur)
|
||||
**Vorteile:**
|
||||
- Extrem schnell und effizient
|
||||
- Exzellent für Microservices
|
||||
- Einfache Deployment (Single Binary)
|
||||
- Gute Concurrency
|
||||
- Statische Typisierung
|
||||
|
||||
**Nachteile:**
|
||||
- Neuer Tech-Stack im Projekt
|
||||
- Getrennte Codebasis von BreakPilot
|
||||
|
||||
#### Option C: Rust (Für maximale Performance & Sicherheit)
|
||||
**Vorteile:**
|
||||
- Höchste Performance
|
||||
- Memory Safety ohne GC
|
||||
- Exzellente Sicherheit
|
||||
- WebAssembly-Support
|
||||
|
||||
**Nachteile:**
|
||||
- Sehr steile Lernkurve
|
||||
- Längere Entwicklungszeit (2-3x)
|
||||
- Kleineres Web-Ecosystem
|
||||
- Komplexere Fehlerbehandlung
|
||||
|
||||
### Finale Empfehlung
|
||||
|
||||
**Für BreakPilot empfehle ich: Go (Golang)**
|
||||
|
||||
Begründung:
|
||||
1. **Unabhängiger Microservice** - Das CMS sollte als eigenständiger Service laufen
|
||||
2. **Performance** - Consent-Checks müssen schnell sein (bei jedem API-Call)
|
||||
3. **Einfaches Deployment** - Single Binary, ideal für Container
|
||||
4. **Gute Balance** - Schneller als Python, einfacher als Rust
|
||||
5. **Zukunftssicher** - Moderne Sprache mit wachsendem Ecosystem
|
||||
|
||||
---
|
||||
|
||||
## Systemarchitektur
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────┐
|
||||
│ BreakPilot Ecosystem │
|
||||
├─────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │
|
||||
│ │ BreakPilot │ │ Consent Admin │ │ BreakPilot │ │
|
||||
│ │ Studio (Web) │ │ Dashboard │ │ Mobile Apps │ │
|
||||
│ │ (Python/HTML) │ │ (Vue.js/React) │ │ (iOS/Android) │ │
|
||||
│ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ │
|
||||
│ │ │ │ │
|
||||
│ └──────────────────────┼──────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌─────────────────────────┐ │
|
||||
│ │ API Gateway / Proxy │ │
|
||||
│ └────────────┬────────────┘ │
|
||||
│ │ │
|
||||
│ ┌─────────────────────┼─────────────────────┐ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │
|
||||
│ │ BreakPilot API │ │ Consent Service │ │ Auth Service │ │
|
||||
│ │ (Python/FastAPI)│ │ (Go) │ │ (Go) │ │
|
||||
│ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ │
|
||||
│ │ │ │ │
|
||||
│ └────────────────────┼────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌─────────────────────────┐ │
|
||||
│ │ PostgreSQL │ │
|
||||
│ │ (Shared Database) │ │
|
||||
│ └─────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Projektphasen
|
||||
|
||||
### Phase 1: Grundlagen & Datenbank (Woche 1-2)
|
||||
**Ziel:** Datenbank-Schema und Basis-Services
|
||||
|
||||
#### 1.1 Datenbank-Design
|
||||
- [ ] Users-Tabelle (Integration mit BreakPilot Auth)
|
||||
- [ ] Legal Documents (AGB, Datenschutz, Community Guidelines, etc.)
|
||||
- [ ] Document Versions (Versionierung mit Freigabe-Workflow)
|
||||
- [ ] User Consents (Welcher User hat wann was zugestimmt)
|
||||
- [ ] Cookie Categories (Notwendig, Funktional, Marketing, Analytics)
|
||||
- [ ] Cookie Consents (Granulare Cookie-Zustimmungen)
|
||||
- [ ] Audit Log (DSGVO-konforme Protokollierung)
|
||||
|
||||
#### 1.2 Go Backend Setup
|
||||
- [ ] Projekt-Struktur mit Clean Architecture
|
||||
- [ ] Database Layer (sqlx oder GORM)
|
||||
- [ ] Migration System
|
||||
- [ ] Config Management
|
||||
- [ ] Logging & Error Handling
|
||||
|
||||
### Phase 2: Core Consent Service (Woche 3-4)
|
||||
**Ziel:** Kern-Funktionalität für Consent-Management
|
||||
|
||||
#### 2.1 Document Management API
|
||||
- [ ] CRUD für Legal Documents
|
||||
- [ ] Versionierung mit Diff-Tracking
|
||||
- [ ] Draft/Published/Archived Status
|
||||
- [ ] Mehrsprachigkeit (DE, EN, etc.)
|
||||
|
||||
#### 2.2 Consent Tracking API
|
||||
- [ ] User Consent erstellen/abrufen
|
||||
- [ ] Consent History pro User
|
||||
- [ ] Bulk-Consent für mehrere Dokumente
|
||||
- [ ] Consent Withdrawal (Widerruf)
|
||||
|
||||
#### 2.3 Cookie Consent API
|
||||
- [ ] Cookie-Kategorien verwalten
|
||||
- [ ] Granulare Cookie-Einstellungen
|
||||
- [ ] Consent-Banner Konfiguration
|
||||
|
||||
### Phase 3: Admin Dashboard (Woche 5-6)
|
||||
**Ziel:** Web-Interface für Administratoren
|
||||
|
||||
#### 3.1 Admin Frontend (Vue.js oder React)
|
||||
- [ ] Login/Auth (Integration mit BreakPilot)
|
||||
- [ ] Dashboard mit Statistiken
|
||||
- [ ] Document Editor (Rich Text)
|
||||
- [ ] Version Management UI
|
||||
- [ ] User Consent Übersicht
|
||||
- [ ] Cookie Management UI
|
||||
|
||||
#### 3.2 Freigabe-Workflow
|
||||
- [ ] Draft → Review → Approved → Published
|
||||
- [ ] Benachrichtigungen bei neuen Versionen
|
||||
- [ ] Rollback-Funktion
|
||||
|
||||
### Phase 4: BreakPilot Integration (Woche 7-8)
|
||||
**Ziel:** Integration in BreakPilot Studio
|
||||
|
||||
#### 4.1 User-facing Features
|
||||
- [ ] "Legal" Button in Einstellungen
|
||||
- [ ] Consent-Historie anzeigen
|
||||
- [ ] Cookie-Präferenzen ändern
|
||||
- [ ] Datenauskunft anfordern (DSGVO Art. 15)
|
||||
|
||||
#### 4.2 Cookie Banner
|
||||
- [ ] Cookie-Consent-Modal beim ersten Besuch
|
||||
- [ ] Granulare Auswahl der Kategorien
|
||||
- [ ] "Alle akzeptieren" / "Nur notwendige"
|
||||
- [ ] Persistente Speicherung
|
||||
|
||||
#### 4.3 Consent-Check Middleware
|
||||
- [ ] Automatische Prüfung bei API-Calls
|
||||
- [ ] Blocking bei fehlender Zustimmung
|
||||
- [ ] Marketing-Opt-out respektieren
|
||||
|
||||
### Phase 5: Data Subject Rights (Woche 9-10)
|
||||
**Ziel:** DSGVO-Compliance Features
|
||||
|
||||
#### 5.1 Datenauskunft (Art. 15 DSGVO)
|
||||
- [ ] API für "Welche Daten haben wir?"
|
||||
- [ ] Export als JSON/PDF
|
||||
- [ ] Automatisierte Bereitstellung
|
||||
|
||||
#### 5.2 Datenlöschung (Art. 17 DSGVO)
|
||||
- [ ] "Recht auf Vergessenwerden"
|
||||
- [ ] Anonymisierung statt Löschung (wo nötig)
|
||||
- [ ] Audit Trail für Löschungen
|
||||
|
||||
#### 5.3 Datenportabilität (Art. 20 DSGVO)
|
||||
- [ ] Export in maschinenlesbarem Format
|
||||
- [ ] Download-Funktion im Frontend
|
||||
|
||||
### Phase 6: Testing & Security (Woche 11-12)
|
||||
**Ziel:** Absicherung und Qualität
|
||||
|
||||
#### 6.1 Testing
|
||||
- [ ] Unit Tests (>80% Coverage)
|
||||
- [ ] Integration Tests
|
||||
- [ ] E2E Tests für kritische Flows
|
||||
- [ ] Performance Tests
|
||||
|
||||
#### 6.2 Security
|
||||
- [ ] Security Audit
|
||||
- [ ] Penetration Testing
|
||||
- [ ] Rate Limiting
|
||||
- [ ] Input Validation
|
||||
- [ ] SQL Injection Prevention
|
||||
- [ ] XSS Protection
|
||||
|
||||
---
|
||||
|
||||
## Datenbank-Schema (Entwurf)
|
||||
|
||||
```sql
|
||||
-- Benutzer (Integration mit BreakPilot)
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
external_id VARCHAR(255) UNIQUE, -- BreakPilot User ID
|
||||
email VARCHAR(255) UNIQUE NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Rechtliche Dokumente
|
||||
CREATE TABLE legal_documents (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
type VARCHAR(50) NOT NULL, -- 'terms', 'privacy', 'cookies', 'community'
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
is_mandatory BOOLEAN DEFAULT true,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Dokumentversionen
|
||||
CREATE TABLE document_versions (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
document_id UUID REFERENCES legal_documents(id) ON DELETE CASCADE,
|
||||
version VARCHAR(20) NOT NULL, -- Semver: 1.0.0, 1.1.0, etc.
|
||||
language VARCHAR(5) DEFAULT 'de', -- ISO 639-1
|
||||
title VARCHAR(255) NOT NULL,
|
||||
content TEXT NOT NULL, -- HTML oder Markdown
|
||||
summary TEXT, -- Kurze Zusammenfassung der Änderungen
|
||||
status VARCHAR(20) DEFAULT 'draft', -- draft, review, approved, published, archived
|
||||
published_at TIMESTAMPTZ,
|
||||
created_by UUID REFERENCES users(id),
|
||||
approved_by UUID REFERENCES users(id),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(document_id, version, language)
|
||||
);
|
||||
|
||||
-- Benutzer-Zustimmungen
|
||||
CREATE TABLE user_consents (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||
document_version_id UUID REFERENCES document_versions(id),
|
||||
consented BOOLEAN NOT NULL,
|
||||
ip_address INET,
|
||||
user_agent TEXT,
|
||||
consented_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
withdrawn_at TIMESTAMPTZ,
|
||||
UNIQUE(user_id, document_version_id)
|
||||
);
|
||||
|
||||
-- Cookie-Kategorien
|
||||
CREATE TABLE cookie_categories (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name VARCHAR(100) NOT NULL, -- 'necessary', 'functional', 'analytics', 'marketing'
|
||||
display_name_de VARCHAR(255) NOT NULL,
|
||||
display_name_en VARCHAR(255),
|
||||
description_de TEXT,
|
||||
description_en TEXT,
|
||||
is_mandatory BOOLEAN DEFAULT false,
|
||||
sort_order INT DEFAULT 0,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Cookie-Zustimmungen
|
||||
CREATE TABLE cookie_consents (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||
category_id UUID REFERENCES cookie_categories(id),
|
||||
consented BOOLEAN NOT NULL,
|
||||
consented_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(user_id, category_id)
|
||||
);
|
||||
|
||||
-- Audit Log (DSGVO-konform)
|
||||
CREATE TABLE consent_audit_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID,
|
||||
action VARCHAR(50) NOT NULL, -- 'consent_given', 'consent_withdrawn', 'data_export', 'data_delete'
|
||||
entity_type VARCHAR(50), -- 'document', 'cookie_category'
|
||||
entity_id UUID,
|
||||
details JSONB,
|
||||
ip_address INET,
|
||||
user_agent TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indizes für Performance
|
||||
CREATE INDEX idx_user_consents_user ON user_consents(user_id);
|
||||
CREATE INDEX idx_user_consents_version ON user_consents(document_version_id);
|
||||
CREATE INDEX idx_cookie_consents_user ON cookie_consents(user_id);
|
||||
CREATE INDEX idx_audit_log_user ON consent_audit_log(user_id);
|
||||
CREATE INDEX idx_audit_log_created ON consent_audit_log(created_at);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## API-Endpoints (Entwurf)
|
||||
|
||||
### Public API (für BreakPilot Frontend)
|
||||
|
||||
```
|
||||
# Dokumente abrufen
|
||||
GET /api/v1/documents # Alle aktiven Dokumente
|
||||
GET /api/v1/documents/:type # Dokument nach Typ (terms, privacy)
|
||||
GET /api/v1/documents/:type/latest # Neueste publizierte Version
|
||||
|
||||
# Consent Management
|
||||
POST /api/v1/consent # Zustimmung erteilen
|
||||
GET /api/v1/consent/my # Meine Zustimmungen
|
||||
GET /api/v1/consent/check/:documentType # Prüfen ob zugestimmt
|
||||
DELETE /api/v1/consent/:id # Zustimmung widerrufen
|
||||
|
||||
# Cookie Consent
|
||||
GET /api/v1/cookies/categories # Cookie-Kategorien
|
||||
POST /api/v1/cookies/consent # Cookie-Präferenzen setzen
|
||||
GET /api/v1/cookies/consent/my # Meine Cookie-Einstellungen
|
||||
|
||||
# Data Subject Rights (DSGVO)
|
||||
GET /api/v1/privacy/my-data # Alle meine Daten abrufen
|
||||
POST /api/v1/privacy/export # Datenexport anfordern
|
||||
POST /api/v1/privacy/delete # Löschung anfordern
|
||||
```
|
||||
|
||||
### Admin API (für Admin Dashboard)
|
||||
|
||||
```
|
||||
# Document Management
|
||||
GET /api/v1/admin/documents # Alle Dokumente (mit Drafts)
|
||||
POST /api/v1/admin/documents # Neues Dokument
|
||||
PUT /api/v1/admin/documents/:id # Dokument bearbeiten
|
||||
DELETE /api/v1/admin/documents/:id # Dokument löschen
|
||||
|
||||
# Version Management
|
||||
GET /api/v1/admin/versions/:docId # Alle Versionen eines Dokuments
|
||||
POST /api/v1/admin/versions # Neue Version erstellen
|
||||
PUT /api/v1/admin/versions/:id # Version bearbeiten
|
||||
POST /api/v1/admin/versions/:id/publish # Version veröffentlichen
|
||||
POST /api/v1/admin/versions/:id/archive # Version archivieren
|
||||
|
||||
# Cookie Categories
|
||||
GET /api/v1/admin/cookies/categories # Alle Kategorien
|
||||
POST /api/v1/admin/cookies/categories # Neue Kategorie
|
||||
PUT /api/v1/admin/cookies/categories/:id
|
||||
DELETE /api/v1/admin/cookies/categories/:id
|
||||
|
||||
# Statistics & Reports
|
||||
GET /api/v1/admin/stats/consents # Consent-Statistiken
|
||||
GET /api/v1/admin/stats/cookies # Cookie-Statistiken
|
||||
GET /api/v1/admin/audit-log # Audit Log (mit Filter)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Consent-Check Middleware (Konzept)
|
||||
|
||||
```go
|
||||
// middleware/consent_check.go
|
||||
|
||||
func ConsentCheckMiddleware(requiredConsent string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
|
||||
// Prüfe ob User zugestimmt hat
|
||||
hasConsent, err := consentService.CheckConsent(userID, requiredConsent)
|
||||
if err != nil {
|
||||
c.AbortWithStatusJSON(500, gin.H{"error": "Consent check failed"})
|
||||
return
|
||||
}
|
||||
|
||||
if !hasConsent {
|
||||
c.AbortWithStatusJSON(403, gin.H{
|
||||
"error": "consent_required",
|
||||
"document_type": requiredConsent,
|
||||
"message": "Sie müssen den Nutzungsbedingungen zustimmen",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// Verwendung in BreakPilot
|
||||
router.POST("/api/worksheets",
|
||||
authMiddleware,
|
||||
ConsentCheckMiddleware("terms"),
|
||||
worksheetHandler.Create,
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Cookie-Banner Flow
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Erster Besuch │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ 1. User öffnet BreakPilot │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ 2. Check: Hat User Cookie-Consent gegeben? │
|
||||
│ │ │
|
||||
│ ┌─────────┴─────────┐ │
|
||||
│ │ Nein │ Ja │
|
||||
│ ▼ ▼ │
|
||||
│ 3. Zeige Cookie Lade gespeicherte │
|
||||
│ Banner Präferenzen │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌─────────────────────────────────────────┐ │
|
||||
│ │ Cookie Consent Banner │ │
|
||||
│ ├─────────────────────────────────────────┤ │
|
||||
│ │ Wir verwenden Cookies, um Ihnen die │ │
|
||||
│ │ beste Erfahrung zu bieten. │ │
|
||||
│ │ │ │
|
||||
│ │ ☑ Notwendig (immer aktiv) │ │
|
||||
│ │ ☐ Funktional │ │
|
||||
│ │ ☐ Analytics │ │
|
||||
│ │ ☐ Marketing │ │
|
||||
│ │ │ │
|
||||
│ │ [Alle akzeptieren] [Auswahl speichern] │ │
|
||||
│ │ [Nur notwendige] [Mehr erfahren] │ │
|
||||
│ └─────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Legal-Bereich im BreakPilot Frontend (Mockup)
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Einstellungen > Legal │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌─────────────────────────────────────────────────────┐ │
|
||||
│ │ Meine Zustimmungen │ │
|
||||
│ ├─────────────────────────────────────────────────────┤ │
|
||||
│ │ │ │
|
||||
│ │ ✓ Allgemeine Geschäftsbedingungen │ │
|
||||
│ │ Version 2.1 · Zugestimmt am 15.11.2024 │ │
|
||||
│ │ [Ansehen] [Widerrufen] │ │
|
||||
│ │ │ │
|
||||
│ │ ✓ Datenschutzerklärung │ │
|
||||
│ │ Version 3.0 · Zugestimmt am 15.11.2024 │ │
|
||||
│ │ [Ansehen] [Widerrufen] │ │
|
||||
│ │ │ │
|
||||
│ │ ✓ Community Guidelines │ │
|
||||
│ │ Version 1.2 · Zugestimmt am 15.11.2024 │ │
|
||||
│ │ [Ansehen] [Widerrufen] │ │
|
||||
│ │ │ │
|
||||
│ └─────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌─────────────────────────────────────────────────────┐ │
|
||||
│ │ Cookie-Einstellungen │ │
|
||||
│ ├─────────────────────────────────────────────────────┤ │
|
||||
│ │ │ │
|
||||
│ │ ☑ Notwendige Cookies (erforderlich) │ │
|
||||
│ │ ☑ Funktionale Cookies │ │
|
||||
│ │ ☐ Analytics Cookies │ │
|
||||
│ │ ☐ Marketing Cookies │ │
|
||||
│ │ │ │
|
||||
│ │ [Einstellungen speichern] │ │
|
||||
│ └─────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌─────────────────────────────────────────────────────┐ │
|
||||
│ │ Meine Daten (DSGVO) │ │
|
||||
│ ├─────────────────────────────────────────────────────┤ │
|
||||
│ │ │ │
|
||||
│ │ [Meine Daten exportieren] │ │
|
||||
│ │ Erhalten Sie eine Kopie aller Ihrer gespeicherten │ │
|
||||
│ │ Daten als JSON-Datei. │ │
|
||||
│ │ │ │
|
||||
│ │ [Account löschen] │ │
|
||||
│ │ Alle Ihre Daten werden unwiderruflich gelöscht. │ │
|
||||
│ │ │ │
|
||||
│ └─────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Nächste Schritte
|
||||
|
||||
### Sofort (diese Woche)
|
||||
1. **Entscheidung:** Go oder Python für Backend?
|
||||
2. **Projekt-Setup:** Repository anlegen
|
||||
3. **Datenbank:** Schema finalisieren und migrieren
|
||||
|
||||
### Kurzfristig (nächste 2 Wochen)
|
||||
1. Core API implementieren
|
||||
2. Basis-Integration in BreakPilot
|
||||
|
||||
### Mittelfristig (nächste 4-6 Wochen)
|
||||
1. Admin Dashboard
|
||||
2. Cookie Banner
|
||||
3. DSGVO-Features
|
||||
|
||||
---
|
||||
|
||||
## Offene Fragen
|
||||
|
||||
1. **Sprache:** Go oder Python für das Backend?
|
||||
2. **Admin Dashboard:** Eigenes Frontend oder in BreakPilot integriert?
|
||||
3. **Hosting:** Gleicher Server wie BreakPilot oder separater Service?
|
||||
4. **Auth:** Shared Authentication mit BreakPilot oder eigenes System?
|
||||
5. **Datenbank:** Shared PostgreSQL oder eigene Instanz?
|
||||
|
||||
---
|
||||
|
||||
## Ressourcen-Schätzung
|
||||
|
||||
| Phase | Aufwand (Tage) | Beschreibung |
|
||||
|-------|---------------|--------------|
|
||||
| Phase 1 | 5-7 | Datenbank & Setup |
|
||||
| Phase 2 | 8-10 | Core Consent Service |
|
||||
| Phase 3 | 10-12 | Admin Dashboard |
|
||||
| Phase 4 | 8-10 | BreakPilot Integration |
|
||||
| Phase 5 | 5-7 | DSGVO Features |
|
||||
| Phase 6 | 5-7 | Testing & Security |
|
||||
| **Gesamt** | **41-53** | ~8-10 Wochen |
|
||||
|
||||
---
|
||||
|
||||
*Dokument erstellt am: 12. Dezember 2024*
|
||||
*Version: 1.0*
|
||||
473
admin-v2/CONTENT_SERVICE_SETUP.md
Normal file
473
admin-v2/CONTENT_SERVICE_SETUP.md
Normal file
@@ -0,0 +1,473 @@
|
||||
# BreakPilot Content Service - Setup & Deployment Guide
|
||||
|
||||
## 🎯 Übersicht
|
||||
|
||||
Der BreakPilot Content Service ist eine vollständige Educational Content Management Plattform mit:
|
||||
|
||||
- ✅ **Content Service API** (FastAPI) - Educational Content Management
|
||||
- ✅ **MinIO S3 Storage** - File Storage für Videos, PDFs, Bilder
|
||||
- ✅ **H5P Service** - Interactive Educational Content (Quizzes, etc.)
|
||||
- ✅ **Matrix Feed Integration** - Content Publishing zu Matrix Spaces
|
||||
- ✅ **PostgreSQL** - Content Metadata Storage
|
||||
- ✅ **Creative Commons Licensing** - CC-BY, CC-BY-SA, etc.
|
||||
- ✅ **Rating & Download Tracking** - Analytics & Impact Scoring
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### 1. Alle Services starten
|
||||
|
||||
```bash
|
||||
# Haupt-Services + Content Services starten
|
||||
docker-compose \
|
||||
-f docker-compose.yml \
|
||||
-f docker-compose.content.yml \
|
||||
up -d
|
||||
|
||||
# Logs verfolgen
|
||||
docker-compose -f docker-compose.yml -f docker-compose.content.yml logs -f
|
||||
```
|
||||
|
||||
### 2. Verfügbare Services
|
||||
|
||||
| Service | URL | Beschreibung |
|
||||
|---------|-----|--------------|
|
||||
| Content Service API | http://localhost:8002 | REST API für Content Management |
|
||||
| MinIO Console | http://localhost:9001 | Storage Dashboard (User: minioadmin, Pass: minioadmin123) |
|
||||
| H5P Service | http://localhost:8003 | Interactive Content Editor |
|
||||
| Content DB | localhost:5433 | PostgreSQL Database |
|
||||
|
||||
### 3. API Dokumentation
|
||||
|
||||
Content Service API Docs:
|
||||
```
|
||||
http://localhost:8002/docs
|
||||
```
|
||||
|
||||
## 📦 Installation (Development)
|
||||
|
||||
### Content Service (Backend)
|
||||
|
||||
```bash
|
||||
cd backend/content_service
|
||||
|
||||
# Virtual Environment erstellen
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate # Windows: venv\Scripts\activate
|
||||
|
||||
# Dependencies installieren
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Environment Variables
|
||||
cp .env.example .env
|
||||
|
||||
# Database Migrations
|
||||
alembic upgrade head
|
||||
|
||||
# Service starten
|
||||
uvicorn main:app --reload --port 8002
|
||||
```
|
||||
|
||||
### H5P Service
|
||||
|
||||
```bash
|
||||
cd h5p-service
|
||||
|
||||
# Dependencies installieren
|
||||
npm install
|
||||
|
||||
# Service starten
|
||||
npm start
|
||||
```
|
||||
|
||||
### Creator Dashboard (Frontend)
|
||||
|
||||
```bash
|
||||
cd frontend/creator-studio
|
||||
|
||||
# Dependencies installieren
|
||||
npm install
|
||||
|
||||
# Development Server
|
||||
npm run dev
|
||||
```
|
||||
|
||||
## 🔧 Konfiguration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Erstelle `.env` im Projekt-Root:
|
||||
|
||||
```env
|
||||
# Content Service
|
||||
CONTENT_DB_URL=postgresql://breakpilot:breakpilot123@localhost:5433/breakpilot_content
|
||||
MINIO_ENDPOINT=localhost:9000
|
||||
MINIO_ACCESS_KEY=minioadmin
|
||||
MINIO_SECRET_KEY=minioadmin123
|
||||
MINIO_BUCKET=breakpilot-content
|
||||
|
||||
# Matrix Integration
|
||||
MATRIX_HOMESERVER=http://localhost:8008
|
||||
MATRIX_ACCESS_TOKEN=your-matrix-token-here
|
||||
MATRIX_BOT_USER=@breakpilot-bot:localhost
|
||||
MATRIX_FEED_ROOM=!breakpilot-feed:localhost
|
||||
|
||||
# OAuth2 (consent-service)
|
||||
CONSENT_SERVICE_URL=http://localhost:8081
|
||||
JWT_SECRET=your-jwt-secret-here
|
||||
|
||||
# H5P Service
|
||||
H5P_BASE_URL=http://localhost:8003
|
||||
H5P_STORAGE_PATH=/app/h5p-content
|
||||
```
|
||||
|
||||
## 📝 Content Service API Endpoints
|
||||
|
||||
### Content Management
|
||||
|
||||
```bash
|
||||
# Create Content
|
||||
POST /api/v1/content
|
||||
{
|
||||
"title": "5-Minuten Yoga für Grundschule",
|
||||
"description": "Bewegungspause mit einfachen Yoga-Übungen",
|
||||
"content_type": "video",
|
||||
"category": "movement",
|
||||
"license": "CC-BY-SA-4.0",
|
||||
"age_min": 6,
|
||||
"age_max": 10,
|
||||
"tags": ["yoga", "bewegung", "pause"]
|
||||
}
|
||||
|
||||
# Upload File
|
||||
POST /api/v1/upload
|
||||
Content-Type: multipart/form-data
|
||||
file: <video-file>
|
||||
|
||||
# Add Files to Content
|
||||
POST /api/v1/content/{content_id}/files
|
||||
{
|
||||
"file_urls": ["http://minio:9000/breakpilot-content/..."]
|
||||
}
|
||||
|
||||
# Publish Content (→ Matrix Feed)
|
||||
POST /api/v1/content/{content_id}/publish
|
||||
|
||||
# List Content (with filters)
|
||||
GET /api/v1/content?category=movement&age_min=6&age_max=10
|
||||
|
||||
# Get Content Details
|
||||
GET /api/v1/content/{content_id}
|
||||
|
||||
# Rate Content
|
||||
POST /api/v1/content/{content_id}/rate
|
||||
{
|
||||
"stars": 5,
|
||||
"comment": "Sehr hilfreich für meine Klasse!"
|
||||
}
|
||||
```
|
||||
|
||||
### H5P Interactive Content
|
||||
|
||||
```bash
|
||||
# Get H5P Editor
|
||||
GET http://localhost:8003/h5p/editor/new
|
||||
|
||||
# Save H5P Content
|
||||
POST http://localhost:8003/h5p/editor
|
||||
{
|
||||
"library": "H5P.InteractiveVideo 1.22",
|
||||
"params": { ... }
|
||||
}
|
||||
|
||||
# Play H5P Content
|
||||
GET http://localhost:8003/h5p/play/{contentId}
|
||||
|
||||
# Export as .h5p File
|
||||
GET http://localhost:8003/h5p/export/{contentId}
|
||||
```
|
||||
|
||||
## 🎨 Creator Workflow
|
||||
|
||||
### 1. Content erstellen
|
||||
|
||||
```javascript
|
||||
// Creator Dashboard
|
||||
const content = await createContent({
|
||||
title: "Mathe-Quiz: Einmaleins",
|
||||
description: "Interaktives Quiz zum Üben des Einmaleins",
|
||||
content_type: "h5p",
|
||||
category: "math",
|
||||
license: "CC-BY-SA-4.0",
|
||||
age_min: 7,
|
||||
age_max: 9
|
||||
});
|
||||
```
|
||||
|
||||
### 2. Files hochladen
|
||||
|
||||
```javascript
|
||||
// Upload Video/PDF/Images
|
||||
const file = document.querySelector('#fileInput').files[0];
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
|
||||
const response = await fetch('/api/v1/upload', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
|
||||
const { file_url } = await response.json();
|
||||
```
|
||||
|
||||
### 3. Publish to Matrix Feed
|
||||
|
||||
```javascript
|
||||
// Publish → Matrix Spaces
|
||||
await publishContent(content.id);
|
||||
// → Content erscheint in #movement, #math, etc.
|
||||
```
|
||||
|
||||
## 📊 Matrix Feed Integration
|
||||
|
||||
### Matrix Spaces Struktur
|
||||
|
||||
```
|
||||
#breakpilot (Root Space)
|
||||
├── #feed (Chronologischer Content Feed)
|
||||
├── #bewegung (Kategorie: Movement)
|
||||
├── #mathe (Kategorie: Math)
|
||||
├── #steam (Kategorie: STEAM)
|
||||
└── #sprache (Kategorie: Language)
|
||||
```
|
||||
|
||||
### Content Message Format
|
||||
|
||||
Wenn Content published wird, erscheint in Matrix:
|
||||
|
||||
```
|
||||
📹 5-Minuten Yoga für Grundschule
|
||||
|
||||
Bewegungspause mit einfachen Yoga-Übungen für den Unterricht
|
||||
|
||||
📝 Von: Max Mustermann
|
||||
🏃 Kategorie: movement
|
||||
👥 Alter: 6-10 Jahre
|
||||
⚖️ Lizenz: CC-BY-SA-4.0
|
||||
🏷️ Tags: yoga, bewegung, pause
|
||||
|
||||
[📥 Inhalt ansehen/herunterladen]
|
||||
```
|
||||
|
||||
## 🔐 Creative Commons Lizenzen
|
||||
|
||||
Verfügbare Lizenzen:
|
||||
|
||||
- `CC-BY-4.0` - Attribution (Namensnennung)
|
||||
- `CC-BY-SA-4.0` - Attribution + ShareAlike (empfohlen)
|
||||
- `CC-BY-NC-4.0` - Attribution + NonCommercial
|
||||
- `CC-BY-NC-SA-4.0` - Attribution + NonCommercial + ShareAlike
|
||||
- `CC0-1.0` - Public Domain
|
||||
|
||||
### Lizenz-Workflow
|
||||
|
||||
```python
|
||||
# Bei Content-Erstellung: Creator wählt Lizenz
|
||||
content.license = "CC-BY-SA-4.0"
|
||||
|
||||
# System validiert:
|
||||
✅ Nur erlaubte Lizenzen
|
||||
✅ Lizenz-Badge wird angezeigt
|
||||
✅ Lizenz-Link zu Creative Commons
|
||||
```
|
||||
|
||||
## 📈 Analytics & Impact Scoring
|
||||
|
||||
### Download Tracking
|
||||
|
||||
```python
|
||||
# Automatisch getrackt bei Download
|
||||
POST /api/v1/content/{content_id}/download
|
||||
|
||||
# → Zähler erhöht
|
||||
# → Download-Event gespeichert
|
||||
# → Für Impact-Score verwendet
|
||||
```
|
||||
|
||||
### Creator Statistics
|
||||
|
||||
```bash
|
||||
# Get Creator Stats
|
||||
GET /api/v1/stats/creator/{creator_id}
|
||||
|
||||
{
|
||||
"total_contents": 12,
|
||||
"total_downloads": 347,
|
||||
"total_views": 1203,
|
||||
"avg_rating": 4.7,
|
||||
"impact_score": 892.5,
|
||||
"content_breakdown": {
|
||||
"movement": 5,
|
||||
"math": 4,
|
||||
"steam": 3
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
### API Tests
|
||||
|
||||
```bash
|
||||
# Pytest
|
||||
cd backend/content_service
|
||||
pytest tests/
|
||||
|
||||
# Mit Coverage
|
||||
pytest --cov=. --cov-report=html
|
||||
```
|
||||
|
||||
### Integration Tests
|
||||
|
||||
```bash
|
||||
# Test Content Upload Flow
|
||||
curl -X POST http://localhost:8002/api/v1/content \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"title": "Test Content",
|
||||
"content_type": "pdf",
|
||||
"category": "math",
|
||||
"license": "CC-BY-SA-4.0"
|
||||
}'
|
||||
```
|
||||
|
||||
## 🐳 Docker Commands
|
||||
|
||||
```bash
|
||||
# Build einzelnen Service
|
||||
docker-compose -f docker-compose.content.yml build content-service
|
||||
|
||||
# Nur Content Services starten
|
||||
docker-compose -f docker-compose.content.yml up -d
|
||||
|
||||
# Logs einzelner Service
|
||||
docker-compose logs -f content-service
|
||||
|
||||
# Service neu starten
|
||||
docker-compose restart content-service
|
||||
|
||||
# Alle stoppen
|
||||
docker-compose -f docker-compose.yml -f docker-compose.content.yml down
|
||||
|
||||
# Mit Volumes löschen (Achtung: Datenverlust!)
|
||||
docker-compose -f docker-compose.yml -f docker-compose.content.yml down -v
|
||||
```
|
||||
|
||||
## 🗄️ Database Migrations
|
||||
|
||||
```bash
|
||||
cd backend/content_service
|
||||
|
||||
# Neue Migration erstellen
|
||||
alembic revision --autogenerate -m "Add new field"
|
||||
|
||||
# Migration anwenden
|
||||
alembic upgrade head
|
||||
|
||||
# Zurückrollen
|
||||
alembic downgrade -1
|
||||
```
|
||||
|
||||
## 📱 Frontend Development
|
||||
|
||||
### Creator Studio
|
||||
|
||||
```bash
|
||||
cd frontend/creator-studio
|
||||
|
||||
# Install dependencies
|
||||
npm install
|
||||
|
||||
# Development
|
||||
npm run dev # → http://localhost:3000
|
||||
|
||||
# Build
|
||||
npm run build
|
||||
|
||||
# Preview Production Build
|
||||
npm run preview
|
||||
```
|
||||
|
||||
## 🔒 DSGVO Compliance
|
||||
|
||||
### Datenminimierung
|
||||
|
||||
- ✅ Nur notwendige Metadaten gespeichert
|
||||
- ✅ Keine Schülerdaten
|
||||
- ✅ IP-Adressen anonymisiert nach 7 Tagen
|
||||
- ✅ User kann Content/Account löschen
|
||||
|
||||
### Datenexport
|
||||
|
||||
```bash
|
||||
# User Data Export
|
||||
GET /api/v1/user/export
|
||||
→ JSON mit allen Daten des Users
|
||||
```
|
||||
|
||||
## 🚨 Troubleshooting
|
||||
|
||||
### MinIO Connection Failed
|
||||
|
||||
```bash
|
||||
# Check MinIO status
|
||||
docker-compose logs minio
|
||||
|
||||
# Test connection
|
||||
curl http://localhost:9000/minio/health/live
|
||||
```
|
||||
|
||||
### Content Service Database Connection
|
||||
|
||||
```bash
|
||||
# Check PostgreSQL
|
||||
docker-compose logs content-db
|
||||
|
||||
# Connect manually
|
||||
docker exec -it breakpilot-pwa-content-db psql -U breakpilot -d breakpilot_content
|
||||
```
|
||||
|
||||
### H5P Service Not Starting
|
||||
|
||||
```bash
|
||||
# Check logs
|
||||
docker-compose logs h5p-service
|
||||
|
||||
# Rebuild
|
||||
docker-compose build h5p-service
|
||||
docker-compose up -d h5p-service
|
||||
```
|
||||
|
||||
## 📚 Weitere Dokumentation
|
||||
|
||||
- [Architekturempfehlung](./backend/docs/Architekturempfehlung%20für%20Breakpilot%20–%20Offene,%20modulare%20Bildungsplattform%20im%20DACH-Raum.pdf)
|
||||
- [Content Service API](./backend/content_service/README.md)
|
||||
- [H5P Integration](./h5p-service/README.md)
|
||||
- [Matrix Feed Setup](./docs/matrix-feed-setup.md)
|
||||
|
||||
## 🎉 Next Steps
|
||||
|
||||
1. ✅ Services starten (siehe Quick Start)
|
||||
2. ✅ Creator Account erstellen
|
||||
3. ✅ Ersten Content hochladen
|
||||
4. ✅ H5P Interactive Content erstellen
|
||||
5. ✅ Content publishen → Matrix Feed
|
||||
6. ✅ Teacher Discovery UI testen
|
||||
7. 🔜 OAuth2 SSO mit consent-service integrieren
|
||||
8. 🔜 Production Deployment vorbereiten
|
||||
|
||||
## 💡 Support
|
||||
|
||||
Bei Fragen oder Problemen:
|
||||
- GitHub Issues: https://github.com/breakpilot/breakpilot-pwa/issues
|
||||
- Matrix Chat: #breakpilot-dev:matrix.org
|
||||
- Email: dev@breakpilot.app
|
||||
427
admin-v2/IMPLEMENTATION_SUMMARY.md
Normal file
427
admin-v2/IMPLEMENTATION_SUMMARY.md
Normal file
@@ -0,0 +1,427 @@
|
||||
# 🎓 BreakPilot Content Service - Implementierungs-Zusammenfassung
|
||||
|
||||
## ✅ Vollständig implementierte Sprints
|
||||
|
||||
### **Sprint 1-2: Content Service Foundation** ✅
|
||||
|
||||
**Backend (FastAPI):**
|
||||
- ✅ Complete Database Schema (PostgreSQL)
|
||||
- `Content` Model mit allen Metadaten
|
||||
- `Rating` Model für Teacher Reviews
|
||||
- `Tag` System für Content Organization
|
||||
- `Download` Tracking für Impact Scoring
|
||||
- ✅ Pydantic Schemas für API Validation
|
||||
- ✅ Full CRUD API für Content Management
|
||||
- ✅ Upload API für Files (Video, PDF, Images, Audio)
|
||||
- ✅ Search & Filter Endpoints
|
||||
- ✅ Analytics & Statistics Endpoints
|
||||
|
||||
**Storage:**
|
||||
- ✅ MinIO S3-kompatible Object Storage
|
||||
- ✅ Automatic Bucket Creation
|
||||
- ✅ Public Read Policy für Content
|
||||
- ✅ File Upload Integration
|
||||
- ✅ Presigned URLs für private Files
|
||||
|
||||
**Files Created:**
|
||||
```
|
||||
backend/content_service/
|
||||
├── models.py # Database Models
|
||||
├── schemas.py # Pydantic Schemas
|
||||
├── database.py # DB Configuration
|
||||
├── main.py # FastAPI Application
|
||||
├── storage.py # MinIO Integration
|
||||
├── requirements.txt # Python Dependencies
|
||||
└── Dockerfile # Container Definition
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### **Sprint 3-4: Matrix Feed Integration** ✅
|
||||
|
||||
**Matrix Client:**
|
||||
- ✅ Matrix SDK Integration (matrix-nio)
|
||||
- ✅ Content Publishing to Matrix Spaces
|
||||
- ✅ Formatted Messages (Plain Text + HTML)
|
||||
- ✅ Category-based Room Routing
|
||||
- ✅ Rich Metadata for Content
|
||||
- ✅ Reactions & Threading Support
|
||||
|
||||
**Matrix Spaces Struktur:**
|
||||
```
|
||||
#breakpilot:server.de (Root Space)
|
||||
├── #feed (Chronologischer Content Feed)
|
||||
├── #bewegung (Movement Category)
|
||||
├── #mathe (Math Category)
|
||||
├── #steam (STEAM Category)
|
||||
└── #sprache (Language Category)
|
||||
```
|
||||
|
||||
**Files Created:**
|
||||
```
|
||||
backend/content_service/
|
||||
└── matrix_client.py # Matrix Integration
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- ✅ Auto-publish on Content.status = PUBLISHED
|
||||
- ✅ Rich HTML Formatting mit Thumbnails
|
||||
- ✅ CC License Badges in Messages
|
||||
- ✅ Direct Links zu Content
|
||||
- ✅ Category-specific Posting
|
||||
|
||||
---
|
||||
|
||||
### **Sprint 5-6: Rating & Download Tracking** ✅
|
||||
|
||||
**Rating System:**
|
||||
- ✅ 5-Star Rating System
|
||||
- ✅ Text Comments
|
||||
- ✅ Average Rating Calculation
|
||||
- ✅ Rating Count Tracking
|
||||
- ✅ One Rating per User (Update möglich)
|
||||
|
||||
**Download Tracking:**
|
||||
- ✅ Event-based Download Logging
|
||||
- ✅ User-specific Tracking
|
||||
- ✅ IP Anonymization (nach 7 Tagen)
|
||||
- ✅ Download Counter
|
||||
- ✅ Impact Score Foundation
|
||||
|
||||
**Analytics:**
|
||||
- ✅ Platform-wide Statistics
|
||||
- ✅ Creator Statistics
|
||||
- ✅ Content Breakdown by Category
|
||||
- ✅ Downloads, Views, Ratings
|
||||
|
||||
---
|
||||
|
||||
### **Sprint 7-8: H5P Interactive Content** ✅
|
||||
|
||||
**H5P Service (Node.js):**
|
||||
- ✅ Self-hosted H5P Server
|
||||
- ✅ H5P Editor Integration
|
||||
- ✅ H5P Player
|
||||
- ✅ File-based Content Storage
|
||||
- ✅ Library Management
|
||||
- ✅ Export as .h5p Files
|
||||
- ✅ Import .h5p Files
|
||||
|
||||
**Supported H5P Content Types:**
|
||||
- ✅ Interactive Video
|
||||
- ✅ Course Presentation
|
||||
- ✅ Quiz (Multiple Choice)
|
||||
- ✅ Drag & Drop
|
||||
- ✅ Timeline
|
||||
- ✅ Memory Game
|
||||
- ✅ Fill in the Blanks
|
||||
- ✅ 50+ weitere Content Types
|
||||
|
||||
**Files Created:**
|
||||
```
|
||||
h5p-service/
|
||||
├── server.js # H5P Express Server
|
||||
├── package.json # Node Dependencies
|
||||
└── Dockerfile # Container Definition
|
||||
```
|
||||
|
||||
**Integration:**
|
||||
- ✅ Content Service → H5P Service API
|
||||
- ✅ H5P Content ID in Content Model
|
||||
- ✅ Automatic Publishing to Matrix
|
||||
|
||||
---
|
||||
|
||||
### **Sprint 7-8: Creative Commons Licensing** ✅
|
||||
|
||||
**Lizenz-System:**
|
||||
- ✅ CC-BY-4.0
|
||||
- ✅ CC-BY-SA-4.0 (Recommended)
|
||||
- ✅ CC-BY-NC-4.0
|
||||
- ✅ CC-BY-NC-SA-4.0
|
||||
- ✅ CC0-1.0 (Public Domain)
|
||||
|
||||
**Features:**
|
||||
- ✅ License Validation bei Upload
|
||||
- ✅ License Selector in Creator Studio
|
||||
- ✅ License Badges in UI
|
||||
- ✅ Direct Links zu Creative Commons
|
||||
- ✅ Matrix Messages mit License Info
|
||||
|
||||
---
|
||||
|
||||
### **Sprint 7-8: DSGVO Compliance** ✅
|
||||
|
||||
**Privacy by Design:**
|
||||
- ✅ Datenminimierung (nur notwendige Daten)
|
||||
- ✅ EU Server Hosting
|
||||
- ✅ IP Anonymization
|
||||
- ✅ User Data Export API
|
||||
- ✅ Account Deletion
|
||||
- ✅ No Schülerdaten
|
||||
|
||||
**Transparency:**
|
||||
- ✅ Clear License Information
|
||||
- ✅ Open Source Code
|
||||
- ✅ Transparent Analytics
|
||||
|
||||
---
|
||||
|
||||
## 🐳 Docker Infrastructure
|
||||
|
||||
**docker-compose.content.yml:**
|
||||
```yaml
|
||||
Services:
|
||||
- minio (Object Storage)
|
||||
- content-db (PostgreSQL)
|
||||
- content-service (FastAPI)
|
||||
- h5p-service (Node.js H5P)
|
||||
|
||||
Volumes:
|
||||
- minio_data
|
||||
- content_db_data
|
||||
- h5p_content
|
||||
|
||||
Networks:
|
||||
- breakpilot-pwa-network (external)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Architektur-Übersicht
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ BREAKPILOT CONTENT PLATFORM │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌───────────┐ │
|
||||
│ │ Creator │───▶│ Content │───▶│ Matrix │ │
|
||||
│ │ Studio │ │ Service │ │ Feed │ │
|
||||
│ │ (Vue.js) │ │ (FastAPI) │ │ (Synapse) │ │
|
||||
│ └──────────────┘ └──────┬───────┘ └───────────┘ │
|
||||
│ │ │
|
||||
│ ┌────────┴────────┐ │
|
||||
│ │ │ │
|
||||
│ ┌──────▼─────┐ ┌─────▼─────┐ │
|
||||
│ │ MinIO │ │ H5P │ │
|
||||
│ │ Storage │ │ Service │ │
|
||||
│ └────────────┘ └───────────┘ │
|
||||
│ │ │ │
|
||||
│ ┌──────▼─────────────────▼─────┐ │
|
||||
│ │ PostgreSQL Database │ │
|
||||
│ └──────────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌───────────┐ │
|
||||
│ │ Teacher │────────────────────────▶│ Content │ │
|
||||
│ │ Discovery │ Search & Download │ Player │ │
|
||||
│ │ UI │ │ │ │
|
||||
│ └──────────────┘ └───────────┘ │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Deployment
|
||||
|
||||
### Quick Start
|
||||
|
||||
```bash
|
||||
# 1. Startup Script ausführbar machen
|
||||
chmod +x scripts/start-content-services.sh
|
||||
|
||||
# 2. Alle Services starten
|
||||
./scripts/start-content-services.sh
|
||||
|
||||
# ODER manuell:
|
||||
docker-compose \
|
||||
-f docker-compose.yml \
|
||||
-f docker-compose.content.yml \
|
||||
up -d
|
||||
```
|
||||
|
||||
### URLs nach Start
|
||||
|
||||
| Service | URL | Credentials |
|
||||
|---------|-----|-------------|
|
||||
| Content Service API | http://localhost:8002/docs | - |
|
||||
| MinIO Console | http://localhost:9001 | minioadmin / minioadmin123 |
|
||||
| H5P Editor | http://localhost:8003/h5p/editor/new | - |
|
||||
| Content Database | localhost:5433 | breakpilot / breakpilot123 |
|
||||
|
||||
---
|
||||
|
||||
## 📝 Content Creation Workflow
|
||||
|
||||
### 1. Creator erstellt Content
|
||||
|
||||
```javascript
|
||||
// POST /api/v1/content
|
||||
{
|
||||
"title": "5-Minuten Yoga",
|
||||
"description": "Bewegungspause für Grundschüler",
|
||||
"content_type": "video",
|
||||
"category": "movement",
|
||||
"license": "CC-BY-SA-4.0",
|
||||
"age_min": 6,
|
||||
"age_max": 10,
|
||||
"tags": ["yoga", "bewegung"]
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Upload Media Files
|
||||
|
||||
```javascript
|
||||
// POST /api/v1/upload
|
||||
FormData {
|
||||
file: <video-file.mp4>
|
||||
}
|
||||
→ Returns: { file_url: "http://minio:9000/..." }
|
||||
```
|
||||
|
||||
### 3. Attach Files to Content
|
||||
|
||||
```javascript
|
||||
// POST /api/v1/content/{id}/files
|
||||
{
|
||||
"file_urls": ["http://minio:9000/..."]
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Publish to Matrix
|
||||
|
||||
```javascript
|
||||
// POST /api/v1/content/{id}/publish
|
||||
→ Status: PUBLISHED
|
||||
→ Matrix Message in #movement Space
|
||||
→ Discoverable by Teachers
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎨 Frontend Components (Creator Studio)
|
||||
|
||||
### Struktur (Vorbereitet)
|
||||
|
||||
```
|
||||
frontend/creator-studio/
|
||||
├── src/
|
||||
│ ├── components/
|
||||
│ │ ├── ContentUpload.vue
|
||||
│ │ ├── ContentList.vue
|
||||
│ │ ├── ContentEditor.vue
|
||||
│ │ ├── H5PEditor.vue
|
||||
│ │ └── Analytics.vue
|
||||
│ ├── views/
|
||||
│ │ ├── Dashboard.vue
|
||||
│ │ ├── CreateContent.vue
|
||||
│ │ └── MyContent.vue
|
||||
│ ├── api/
|
||||
│ │ └── content.js
|
||||
│ └── router/
|
||||
│ └── index.js
|
||||
├── package.json
|
||||
└── vite.config.js
|
||||
```
|
||||
|
||||
**Status:** Framework vorbereitet, vollständige UI-Implementation ausstehend (Sprint 1-2 Frontend)
|
||||
|
||||
---
|
||||
|
||||
## ⏭️ Nächste Schritte (Optional/Future)
|
||||
|
||||
### **Ausstehend:**
|
||||
|
||||
1. **OAuth2 SSO Integration** (Sprint 3-4)
|
||||
- consent-service → Matrix SSO
|
||||
- JWT Validation in Content Service
|
||||
- User Roles & Permissions
|
||||
|
||||
2. **Teacher Discovery UI** (Sprint 5-6)
|
||||
- Vue.js Frontend komplett
|
||||
- Search & Filter UI
|
||||
- Content Preview & Download
|
||||
- Rating Interface
|
||||
|
||||
3. **Production Deployment**
|
||||
- Environment Configuration
|
||||
- SSL/TLS Certificates
|
||||
- Backup Strategy
|
||||
- Monitoring (Prometheus/Grafana)
|
||||
|
||||
---
|
||||
|
||||
## 📈 Impact Scoring (Fundament gelegt)
|
||||
|
||||
**Vorbereitet für zukünftige Implementierung:**
|
||||
|
||||
```python
|
||||
# Impact Score Calculation (Beispiel)
|
||||
impact_score = (
|
||||
downloads * 10 +
|
||||
rating_count * 5 +
|
||||
avg_rating * 20 +
|
||||
matrix_engagement * 2
|
||||
)
|
||||
```
|
||||
|
||||
**Bereits getrackt:**
|
||||
- ✅ Downloads
|
||||
- ✅ Views
|
||||
- ✅ Ratings (Stars + Comments)
|
||||
- ✅ Matrix Event IDs
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Erreichte Features (Zusammenfassung)
|
||||
|
||||
| Feature | Status | Sprint |
|
||||
|---------|--------|--------|
|
||||
| Content CRUD API | ✅ | 1-2 |
|
||||
| File Upload (MinIO) | ✅ | 1-2 |
|
||||
| PostgreSQL Schema | ✅ | 1-2 |
|
||||
| Matrix Feed Publishing | ✅ | 3-4 |
|
||||
| Rating System | ✅ | 5-6 |
|
||||
| Download Tracking | ✅ | 5-6 |
|
||||
| H5P Integration | ✅ | 7-8 |
|
||||
| CC Licensing | ✅ | 7-8 |
|
||||
| DSGVO Compliance | ✅ | 7-8 |
|
||||
| Docker Setup | ✅ | 7-8 |
|
||||
| Deployment Guide | ✅ | 7-8 |
|
||||
| Creator Studio (Backend) | ✅ | 1-2 |
|
||||
| Creator Studio (Frontend) | 🔜 | Pending |
|
||||
| Teacher Discovery UI | 🔜 | Pending |
|
||||
| OAuth2 SSO | 🔜 | Pending |
|
||||
|
||||
---
|
||||
|
||||
## 📚 Dokumentation
|
||||
|
||||
- ✅ **CONTENT_SERVICE_SETUP.md** - Vollständiger Setup Guide
|
||||
- ✅ **IMPLEMENTATION_SUMMARY.md** - Diese Datei
|
||||
- ✅ **API Dokumentation** - Auto-generiert via FastAPI (/docs)
|
||||
- ✅ **Architekturempfehlung PDF** - Strategische Planung
|
||||
|
||||
---
|
||||
|
||||
## 🎉 Fazit
|
||||
|
||||
**Implementiert:** 8+ Wochen Entwicklung in Sprints 1-8
|
||||
|
||||
**Kernfunktionen:**
|
||||
- ✅ Vollständiger Content Service (Backend)
|
||||
- ✅ MinIO S3 Storage
|
||||
- ✅ H5P Interactive Content
|
||||
- ✅ Matrix Feed Integration
|
||||
- ✅ Creative Commons Licensing
|
||||
- ✅ Rating & Analytics
|
||||
- ✅ DSGVO Compliance
|
||||
- ✅ Docker Deployment Ready
|
||||
|
||||
**Ready to Use:** Alle Backend-Services produktionsbereit
|
||||
|
||||
**Next:** Frontend UI vervollständigen & Production Deploy
|
||||
|
||||
---
|
||||
|
||||
**🚀 Die BreakPilot Content Platform ist LIVE!**
|
||||
95
admin-v2/MAC_MINI_SETUP.md
Normal file
95
admin-v2/MAC_MINI_SETUP.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# Mac Mini Headless Setup - Vollständig Automatisch
|
||||
|
||||
## Verbindungsdaten
|
||||
- **IP (LAN):** 192.168.178.100
|
||||
- **IP (WiFi):** 192.168.178.163 (nicht mehr aktiv)
|
||||
- **User:** benjaminadmin
|
||||
- **SSH:** `ssh benjaminadmin@192.168.178.100`
|
||||
|
||||
## Nach Neustart - Alles startet automatisch!
|
||||
|
||||
| Service | Auto-Start | Port |
|
||||
|---------|------------|------|
|
||||
| ✅ SSH | Ja | 22 |
|
||||
| ✅ Docker Desktop | Ja | - |
|
||||
| ✅ Docker Container | Ja (nach ~2 Min) | 8000, 8081, etc. |
|
||||
| ✅ Ollama Server | Ja | 11434 |
|
||||
| ✅ Unity Hub | Ja | - |
|
||||
| ✅ VS Code | Ja | - |
|
||||
|
||||
**Keine Aktion nötig nach Neustart!** Einfach 2-3 Minuten warten.
|
||||
|
||||
## Status prüfen
|
||||
```bash
|
||||
./scripts/mac-mini/status.sh
|
||||
```
|
||||
|
||||
## Services & Ports
|
||||
| Service | Port | URL |
|
||||
|---------|------|-----|
|
||||
| Backend API | 8000 | http://192.168.178.100:8000/admin |
|
||||
| Consent Service | 8081 | - |
|
||||
| PostgreSQL | 5432 | - |
|
||||
| Valkey/Redis | 6379 | - |
|
||||
| MinIO | 9000/9001 | http://192.168.178.100:9001 |
|
||||
| Mailpit | 8025 | http://192.168.178.100:8025 |
|
||||
| Ollama | 11434 | http://192.168.178.100:11434/api/tags |
|
||||
|
||||
## LLM Modelle
|
||||
- **Qwen 2.5 14B** (14.8 Milliarden Parameter)
|
||||
|
||||
## Scripts (auf MacBook)
|
||||
```bash
|
||||
./scripts/mac-mini/status.sh # Status prüfen
|
||||
./scripts/mac-mini/sync.sh # Code synchronisieren
|
||||
./scripts/mac-mini/docker.sh # Docker-Befehle
|
||||
./scripts/mac-mini/backup.sh # Backup erstellen
|
||||
```
|
||||
|
||||
## Docker-Befehle
|
||||
```bash
|
||||
./scripts/mac-mini/docker.sh ps # Container anzeigen
|
||||
./scripts/mac-mini/docker.sh logs backend # Logs
|
||||
./scripts/mac-mini/docker.sh restart # Neustart
|
||||
./scripts/mac-mini/docker.sh build # Image bauen
|
||||
```
|
||||
|
||||
## LaunchAgents (Auto-Start)
|
||||
Pfad auf Mac Mini: `~/Library/LaunchAgents/`
|
||||
|
||||
| Agent | Funktion |
|
||||
|-------|----------|
|
||||
| `com.docker.desktop.plist` | Docker Desktop |
|
||||
| `com.breakpilot.docker-containers.plist` | Container Auto-Start |
|
||||
| `com.ollama.serve.plist` | Ollama Server |
|
||||
| `com.unity.hub.plist` | Unity Hub |
|
||||
| `com.microsoft.vscode.plist` | VS Code |
|
||||
|
||||
## Projekt-Pfade
|
||||
- **MacBook:** `/Users/benjaminadmin/Projekte/breakpilot-pwa/`
|
||||
- **Mac Mini:** `/Users/benjaminadmin/Projekte/breakpilot-pwa/`
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Docker Onboarding erscheint wieder
|
||||
Docker-Einstellungen sind gesichert in `~/docker-settings-backup/`
|
||||
```bash
|
||||
# Wiederherstellen:
|
||||
cp -r ~/docker-settings-backup/* ~/Library/Group\ Containers/group.com.docker/
|
||||
```
|
||||
|
||||
### Container starten nicht automatisch
|
||||
Log prüfen:
|
||||
```bash
|
||||
ssh benjaminadmin@192.168.178.163 "cat /tmp/docker-autostart.log"
|
||||
```
|
||||
|
||||
Manuell starten:
|
||||
```bash
|
||||
./scripts/mac-mini/docker.sh up
|
||||
```
|
||||
|
||||
### SSH nicht erreichbar
|
||||
- Prüfe ob Mac Mini an ist (Ping: `ping 192.168.178.163`)
|
||||
- Warte 1-2 Minuten nach Boot
|
||||
- Prüfe Netzwerkverbindung
|
||||
80
admin-v2/Makefile
Normal file
80
admin-v2/Makefile
Normal file
@@ -0,0 +1,80 @@
|
||||
# BreakPilot PWA - Makefile fuer lokale CI-Simulation
|
||||
#
|
||||
# Verwendung:
|
||||
# make ci - Alle Tests lokal ausfuehren
|
||||
# make test-go - Nur Go-Tests
|
||||
# make test-python - Nur Python-Tests
|
||||
# make logs-agent - Woodpecker Agent Logs
|
||||
# make logs-backend - Backend Logs (ci-result)
|
||||
|
||||
.PHONY: ci test-go test-python test-node logs-agent logs-backend clean help
|
||||
|
||||
# Verzeichnis fuer Test-Ergebnisse
|
||||
CI_RESULTS_DIR := .ci-results
|
||||
|
||||
help:
|
||||
@echo "BreakPilot CI - Verfuegbare Befehle:"
|
||||
@echo ""
|
||||
@echo " make ci - Alle Tests lokal ausfuehren"
|
||||
@echo " make test-go - Go Service Tests"
|
||||
@echo " make test-python - Python Service Tests"
|
||||
@echo " make test-node - Node.js Service Tests"
|
||||
@echo " make logs-agent - Woodpecker Agent Logs anzeigen"
|
||||
@echo " make logs-backend - Backend Logs (ci-result) anzeigen"
|
||||
@echo " make clean - Test-Ergebnisse loeschen"
|
||||
|
||||
ci: test-go test-python test-node
|
||||
@echo "========================================="
|
||||
@echo "Local CI complete. Results in $(CI_RESULTS_DIR)/"
|
||||
@echo "========================================="
|
||||
@ls -la $(CI_RESULTS_DIR)/
|
||||
|
||||
test-go: $(CI_RESULTS_DIR)
|
||||
@echo "=== Go Tests ==="
|
||||
@if [ -d "consent-service" ]; then \
|
||||
cd consent-service && go test -v -json ./... > ../$(CI_RESULTS_DIR)/test-consent.json 2>&1 || true; \
|
||||
echo "consent-service: done"; \
|
||||
fi
|
||||
@if [ -d "billing-service" ]; then \
|
||||
cd billing-service && go test -v -json ./... > ../$(CI_RESULTS_DIR)/test-billing.json 2>&1 || true; \
|
||||
echo "billing-service: done"; \
|
||||
fi
|
||||
@if [ -d "school-service" ]; then \
|
||||
cd school-service && go test -v -json ./... > ../$(CI_RESULTS_DIR)/test-school.json 2>&1 || true; \
|
||||
echo "school-service: done"; \
|
||||
fi
|
||||
|
||||
test-python: $(CI_RESULTS_DIR)
|
||||
@echo "=== Python Tests ==="
|
||||
@if [ -d "backend" ]; then \
|
||||
cd backend && python -m pytest tests/ -v --tb=short 2>&1 || true; \
|
||||
echo "backend: done"; \
|
||||
fi
|
||||
@if [ -d "voice-service" ]; then \
|
||||
cd voice-service && python -m pytest tests/ -v --tb=short 2>&1 || true; \
|
||||
echo "voice-service: done"; \
|
||||
fi
|
||||
@if [ -d "klausur-service/backend" ]; then \
|
||||
cd klausur-service/backend && python -m pytest tests/ -v --tb=short 2>&1 || true; \
|
||||
echo "klausur-service: done"; \
|
||||
fi
|
||||
|
||||
test-node: $(CI_RESULTS_DIR)
|
||||
@echo "=== Node.js Tests ==="
|
||||
@if [ -d "h5p-service" ]; then \
|
||||
cd h5p-service && npm test 2>&1 || true; \
|
||||
echo "h5p-service: done"; \
|
||||
fi
|
||||
|
||||
$(CI_RESULTS_DIR):
|
||||
@mkdir -p $(CI_RESULTS_DIR)
|
||||
|
||||
logs-agent:
|
||||
docker logs breakpilot-pwa-woodpecker-agent --tail=200
|
||||
|
||||
logs-backend:
|
||||
docker compose logs backend --tail=200 | grep -E "(ci-result|error|ERROR)"
|
||||
|
||||
clean:
|
||||
rm -rf $(CI_RESULTS_DIR)
|
||||
@echo "Test-Ergebnisse geloescht"
|
||||
794
admin-v2/POLICY_VAULT_OVERVIEW.md
Normal file
794
admin-v2/POLICY_VAULT_OVERVIEW.md
Normal file
@@ -0,0 +1,794 @@
|
||||
# Policy Vault - Projekt-Dokumentation
|
||||
|
||||
## Projektübersicht
|
||||
|
||||
**Policy Vault** ist eine vollständige Web-Anwendung zur Verwaltung von Datenschutzrichtlinien, Cookie-Einwilligungen und Nutzerzustimmungen für verschiedene Projekte und Plattformen. Das System ermöglicht es Administratoren, Datenschutzdokumente zu erstellen, zu verwalten und zu versionieren, sowie Nutzereinwilligungen zu verfolgen und Cookie-Präferenzen zu speichern.
|
||||
|
||||
## Zweck und Anwendungsbereich
|
||||
|
||||
Das Policy Vault System dient als zentrale Plattform für:
|
||||
- **Verwaltung von Datenschutzrichtlinien** (Privacy Policies, Terms of Service, etc.)
|
||||
- **Cookie-Consent-Management** mit Kategorisierung und Vendor-Verwaltung
|
||||
- **Versionskontrolle** für Richtliniendokumente
|
||||
- **Multi-Projekt-Verwaltung** mit rollenbasiertem Zugriff
|
||||
- **Nutzereinwilligungs-Tracking** über verschiedene Plattformen hinweg
|
||||
- **Mehrsprachige Unterstützung** für globale Anwendungen
|
||||
|
||||
---
|
||||
|
||||
## Technologie-Stack
|
||||
|
||||
### Backend
|
||||
- **Framework**: NestJS (Node.js/TypeScript)
|
||||
- **Datenbank**: PostgreSQL
|
||||
- **ORM**: Drizzle ORM
|
||||
- **Authentifizierung**: JWT (JSON Web Tokens) mit Access/Refresh Token
|
||||
- **API-Dokumentation**: Swagger/OpenAPI
|
||||
- **Validierung**: class-validator, class-transformer
|
||||
- **Security**:
|
||||
- Encryption-based authentication
|
||||
- Rate limiting (Throttler)
|
||||
- Role-based access control (RBAC)
|
||||
- bcrypt für Password-Hashing
|
||||
- **Logging**: Winston mit Daily Rotate File
|
||||
- **Job Scheduling**: NestJS Schedule
|
||||
- **E-Mail**: Nodemailer
|
||||
- **OTP-Generierung**: otp-generator
|
||||
|
||||
### Frontend
|
||||
- **Framework**: Angular 18
|
||||
- **UI**:
|
||||
- TailwindCSS
|
||||
- Custom SCSS
|
||||
- **Rich Text Editor**: CKEditor 5
|
||||
- Alignment, Block Quote, Code Block
|
||||
- Font styling, Image support
|
||||
- List und Table support
|
||||
- **State Management**: RxJS
|
||||
- **Security**: DOMPurify für HTML-Sanitization
|
||||
- **Multi-Select**: ng-multiselect-dropdown
|
||||
- **Process Manager**: PM2
|
||||
|
||||
---
|
||||
|
||||
## Hauptfunktionen und Features
|
||||
|
||||
### 1. Administratoren-Verwaltung
|
||||
- **Super Admin und Admin Rollen**
|
||||
- Super Admin (Role 1): Vollzugriff auf alle Funktionen
|
||||
- Admin (Role 2): Eingeschränkter Zugriff auf zugewiesene Projekte
|
||||
- **Authentifizierung**
|
||||
- Login mit E-Mail und Passwort
|
||||
- JWT-basierte Sessions (Access + Refresh Token)
|
||||
- OTP-basierte Passwort-Wiederherstellung
|
||||
- Account-Lock-Mechanismus bei mehrfachen Fehlversuchen
|
||||
- **Benutzerverwaltung**
|
||||
- Admin-Erstellung durch Super Admin
|
||||
- Projekt-Zuweisungen für Admins
|
||||
- Rollen-Modifikation (Promote/Demote)
|
||||
- Soft-Delete (isDeleted Flag)
|
||||
|
||||
### 2. Projekt-Management
|
||||
- **Projektverwaltung**
|
||||
- Erstellung und Verwaltung von Projekten
|
||||
- Projekt-spezifische Konfiguration (Theme-Farben, Icons, Logos)
|
||||
- Mehrsprachige Unterstützung (Language Configuration)
|
||||
- Projekt-Keys für sichere API-Zugriffe
|
||||
- Soft-Delete und Blocking von Projekten
|
||||
- **Projekt-Zugriffskontrolle**
|
||||
- Zuweisung von Admins zu spezifischen Projekten
|
||||
- Project-Admin-Beziehungen
|
||||
|
||||
### 3. Policy Document Management
|
||||
- **Dokumentenverwaltung**
|
||||
- Erstellung von Datenschutzdokumenten (Privacy Policies, ToS, etc.)
|
||||
- Projekt-spezifische Dokumente
|
||||
- Beschreibung und Metadaten
|
||||
- **Versionierung**
|
||||
- Multiple Versionen pro Dokument
|
||||
- Version-Metadaten mit Inhalt
|
||||
- Publish/Draft-Status
|
||||
- Versionsnummern-Tracking
|
||||
|
||||
### 4. Cookie-Consent-Management
|
||||
- **Cookie-Kategorien**
|
||||
- Kategorien-Metadaten (z.B. Notwendig, Marketing, Analytics)
|
||||
- Plattform-spezifische Kategorien (Web, Mobile, etc.)
|
||||
- Versionierung der Kategorien
|
||||
- Pflicht- und optionale Kategorien
|
||||
- Mehrsprachige Kategorie-Beschreibungen
|
||||
- **Vendor-Management**
|
||||
- Verwaltung von Drittanbieter-Services
|
||||
- Vendor-Metadaten und -Beschreibungen
|
||||
- Zuordnung zu Kategorien
|
||||
- Sub-Services für Vendors
|
||||
- Mehrsprachige Vendor-Informationen
|
||||
- **Globale Cookie-Einstellungen**
|
||||
- Projekt-weite Cookie-Texte und -Beschreibungen
|
||||
- Mehrsprachige globale Inhalte
|
||||
- Datei-Upload-Unterstützung
|
||||
|
||||
### 5. User Consent Tracking
|
||||
- **Policy Document Consent**
|
||||
- Tracking von Nutzereinwilligungen für Richtlinien-Versionen
|
||||
- Username-basiertes Tracking
|
||||
- Status (Akzeptiert/Abgelehnt)
|
||||
- Timestamp-Tracking
|
||||
- **Cookie Consent**
|
||||
- Granulare Cookie-Einwilligungen pro Kategorie
|
||||
- Vendor-spezifische Einwilligungen
|
||||
- Versions-Tracking
|
||||
- Username und Projekt-basiert
|
||||
- **Verschlüsselte API-Zugriffe**
|
||||
- Token-basierte Authentifizierung für Mobile/Web
|
||||
- Encryption-based authentication für externe Zugriffe
|
||||
|
||||
### 6. Mehrsprachige Unterstützung
|
||||
- **Language Management**
|
||||
- Dynamische Sprachen-Konfiguration pro Projekt
|
||||
- Mehrsprachige Inhalte für:
|
||||
- Kategorien-Beschreibungen
|
||||
- Vendor-Informationen
|
||||
- Globale Cookie-Texte
|
||||
- Sub-Service-Beschreibungen
|
||||
|
||||
---
|
||||
|
||||
## API-Struktur und Endpoints
|
||||
|
||||
### Admin-Endpoints (`/admins`)
|
||||
```
|
||||
POST /admins/create-admin - Admin erstellen (Super Admin only)
|
||||
POST /admins/create-super-admin - Super Admin erstellen (Super Admin only)
|
||||
POST /admins/create-root-user-super-admin - Root Super Admin erstellen (Secret-based)
|
||||
POST /admins/login - Admin Login
|
||||
GET /admins/get-access-token - Neuen Access Token abrufen
|
||||
POST /admins/generate-otp - OTP für Passwort-Reset generieren
|
||||
POST /admins/validate-otp - OTP validieren
|
||||
POST /admins/change-password - Passwort ändern (mit OTP)
|
||||
PUT /admins/update-password - Passwort aktualisieren (eingeloggt)
|
||||
PUT /admins/forgot-password - Passwort vergessen
|
||||
PUT /admins/make-super-admin - Admin zu Super Admin befördern
|
||||
PUT /admins/remove-super-admin - Super Admin zu Admin zurückstufen
|
||||
PUT /admins/make-project-admin - Projekt-Zugriff gewähren
|
||||
DELETE /admins/remove-project-admin - Projekt-Zugriff entfernen
|
||||
GET /admins/findAll?role= - Alle Admins abrufen (gefiltert nach Rolle)
|
||||
GET /admins/findAll-super-admins - Alle Super Admins abrufen
|
||||
GET /admins/findOne?id= - Einzelnen Admin abrufen
|
||||
PUT /admins/update - Admin-Details aktualisieren
|
||||
DELETE /admins/delete-admin?id= - Admin löschen (Soft-Delete)
|
||||
```
|
||||
|
||||
### Project-Endpoints (`/project`)
|
||||
```
|
||||
POST /project/create - Projekt erstellen (Super Admin only)
|
||||
PUT /project/v2/updateProjectKeys - Projekt-Keys aktualisieren
|
||||
GET /project/findAll - Alle Projekte abrufen (mit Pagination)
|
||||
GET /project/findAllByUser - Projekte eines bestimmten Users
|
||||
GET /project/findOne?id= - Einzelnes Projekt abrufen
|
||||
PUT /project/update - Projekt aktualisieren
|
||||
DELETE /project/delete?id= - Projekt löschen
|
||||
```
|
||||
|
||||
### Policy Document-Endpoints (`/policydocument`)
|
||||
```
|
||||
POST /policydocument/create - Policy Document erstellen
|
||||
GET /policydocument/findAll - Alle Policy Documents abrufen
|
||||
GET /policydocument/findOne?id= - Einzelnes Policy Document
|
||||
GET /policydocument/findPolicyDocs?projectId= - Documents für ein Projekt
|
||||
PUT /policydocument/update - Policy Document aktualisieren
|
||||
DELETE /policydocument/delete?id= - Policy Document löschen
|
||||
```
|
||||
|
||||
### Version-Endpoints (`/version`)
|
||||
```
|
||||
POST /version/create - Version erstellen
|
||||
GET /version/findAll - Alle Versionen abrufen
|
||||
GET /version/findOne?id= - Einzelne Version abrufen
|
||||
GET /version/findVersions?policyDocId= - Versionen für ein Policy Document
|
||||
PUT /version/update - Version aktualisieren
|
||||
DELETE /version/delete?id= - Version löschen
|
||||
```
|
||||
|
||||
### User Consent-Endpoints (`/consent`)
|
||||
```
|
||||
POST /consent/v2/create - User Consent erstellen (Encrypted)
|
||||
GET /consent/v2/GetConsent - Consent abrufen (Encrypted)
|
||||
GET /consent/v2/GetConsentFileContent - Consent mit Dateiinhalt (Encrypted)
|
||||
GET /consent/v2/latestAcceptedConsent - Letzte akzeptierte Consent
|
||||
DELETE /consent/v2/delete - Consent löschen (Encrypted)
|
||||
```
|
||||
|
||||
### Cookie Consent-Endpoints (`/cookieconsent`)
|
||||
```
|
||||
POST /cookieconsent/v2/create - Cookie Consent erstellen (Encrypted)
|
||||
GET /cookieconsent/v2/get - Cookie Kategorien abrufen (Encrypted)
|
||||
GET /cookieconsent/v2/getFileContent - Cookie Daten mit Dateiinhalt (Encrypted)
|
||||
DELETE /cookieconsent/v2/delete - Cookie Consent löschen (Encrypted)
|
||||
```
|
||||
|
||||
### Cookie-Endpoints (`/cookies`)
|
||||
```
|
||||
POST /cookies/createCategory - Cookie-Kategorie erstellen
|
||||
POST /cookies/createVendor - Vendor erstellen
|
||||
POST /cookies/createGlobalCookie - Globale Cookie-Einstellung erstellen
|
||||
GET /cookies/getCategories?projectId= - Kategorien für Projekt abrufen
|
||||
GET /cookies/getVendors?projectId= - Vendors für Projekt abrufen
|
||||
GET /cookies/getGlobalCookie?projectId= - Globale Cookie-Settings
|
||||
PUT /cookies/updateCategory - Kategorie aktualisieren
|
||||
PUT /cookies/updateVendor - Vendor aktualisieren
|
||||
PUT /cookies/updateGlobalCookie - Globale Settings aktualisieren
|
||||
DELETE /cookies/deleteCategory?id= - Kategorie löschen
|
||||
DELETE /cookies/deleteVendor?id= - Vendor löschen
|
||||
DELETE /cookies/deleteGlobalCookie?id= - Globale Settings löschen
|
||||
```
|
||||
|
||||
### Health Check-Endpoint (`/db-health-check`)
|
||||
```
|
||||
GET /db-health-check - Datenbank-Status prüfen
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Datenmodelle
|
||||
|
||||
### Admin
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
employeeCode: string (nullable)
|
||||
firstName: string (max 60)
|
||||
lastName: string (max 50)
|
||||
officialMail: string (unique, max 100)
|
||||
role: number (1 = Super Admin, 2 = Admin)
|
||||
passwordHash: string
|
||||
salt: string (nullable)
|
||||
accessToken: text (nullable)
|
||||
refreshToken: text (nullable)
|
||||
accLockCount: number (default 0)
|
||||
accLockTime: number (default 0)
|
||||
isBlocked: boolean (default false)
|
||||
isDeleted: boolean (default false)
|
||||
otp: string (nullable)
|
||||
}
|
||||
```
|
||||
|
||||
### Project
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
name: string (unique)
|
||||
description: string
|
||||
imageURL: text (nullable)
|
||||
iconURL: text (nullable)
|
||||
isBlocked: boolean (default false)
|
||||
isDeleted: boolean (default false)
|
||||
themeColor: string
|
||||
textColor: string
|
||||
languages: json (nullable) // Array von Sprach-Codes
|
||||
}
|
||||
```
|
||||
|
||||
### Policy Document
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
name: string
|
||||
description: string (nullable)
|
||||
projectId: number (FK -> project.id, CASCADE)
|
||||
}
|
||||
```
|
||||
|
||||
### Version (Policy Document Meta & Version Meta)
|
||||
```typescript
|
||||
// Policy Document Meta
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
policyDocumentId: number (FK)
|
||||
version: string
|
||||
isPublish: boolean
|
||||
}
|
||||
|
||||
// Version Meta (Sprachspezifischer Inhalt)
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
policyDocMetaId: number (FK)
|
||||
language: string
|
||||
content: text
|
||||
file: text (nullable)
|
||||
}
|
||||
```
|
||||
|
||||
### User Consent
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
username: string
|
||||
status: boolean
|
||||
projectId: number (FK -> project.id, CASCADE)
|
||||
versionMetaId: number (FK -> versionMeta.id, CASCADE)
|
||||
}
|
||||
```
|
||||
|
||||
### Cookie Consent
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
username: string
|
||||
categoryId: number[] (Array)
|
||||
vendors: number[] (Array)
|
||||
projectId: number (FK -> project.id, CASCADE)
|
||||
version: string
|
||||
}
|
||||
```
|
||||
|
||||
### Categories Metadata
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
projectId: number (FK -> project.id, CASCADE)
|
||||
platform: string
|
||||
version: string
|
||||
isPublish: boolean (default false)
|
||||
metaName: string
|
||||
isMandatory: boolean (default false)
|
||||
}
|
||||
```
|
||||
|
||||
### Categories Language Data
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
categoryMetaId: number (FK -> categoriesMetadata.id, CASCADE)
|
||||
language: string
|
||||
title: string
|
||||
description: text
|
||||
}
|
||||
```
|
||||
|
||||
### Vendor Meta
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
categoryId: number (FK -> categoriesMetadata.id, CASCADE)
|
||||
vendorName: string
|
||||
}
|
||||
```
|
||||
|
||||
### Vendor Language
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
vendorMetaId: number (FK -> vendorMeta.id, CASCADE)
|
||||
language: string
|
||||
description: text
|
||||
}
|
||||
```
|
||||
|
||||
### Sub Service
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
vendorMetaId: number (FK -> vendorMeta.id, CASCADE)
|
||||
serviceName: string
|
||||
}
|
||||
```
|
||||
|
||||
### Global Cookie Metadata
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
projectId: number (FK -> project.id, CASCADE)
|
||||
version: string
|
||||
isPublish: boolean (default false)
|
||||
}
|
||||
```
|
||||
|
||||
### Global Cookie Language Data
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
globalCookieMetaId: number (FK -> globalCookieMetadata.id, CASCADE)
|
||||
language: string
|
||||
title: string
|
||||
description: text
|
||||
file: text (nullable)
|
||||
}
|
||||
```
|
||||
|
||||
### Project Keys
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
createdAt: timestamp
|
||||
updatedAt: timestamp
|
||||
projectId: number (FK -> project.id, CASCADE)
|
||||
publicKey: text
|
||||
privateKey: text
|
||||
}
|
||||
```
|
||||
|
||||
### Admin Projects (Junction Table)
|
||||
```typescript
|
||||
{
|
||||
id: number (PK)
|
||||
adminId: number (FK -> admin.id, CASCADE)
|
||||
projectId: number (FK -> project.id, CASCADE)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Architektur-Übersicht
|
||||
|
||||
### Backend-Architektur
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ NestJS Backend │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ Guards │ │ Middlewares │ │ Interceptors │ │
|
||||
│ ├──────────────┤ ├──────────────┤ ├──────────────┤ │
|
||||
│ │ - AuthGuard │ │ - Token │ │ - Serialize │ │
|
||||
│ │ - RolesGuard │ │ - Decrypt │ │ - Logging │ │
|
||||
│ │ - Throttler │ │ - Headers │ │ │ │
|
||||
│ └──────────────┘ └──────────────┘ └──────────────┘ │
|
||||
│ │
|
||||
│ ┌───────────────────────────────────────────────────┐ │
|
||||
│ │ API Modules │ │
|
||||
│ ├───────────────────────────────────────────────────┤ │
|
||||
│ │ - Admins (Authentication, Authorization) │ │
|
||||
│ │ - Projects (Multi-tenant Management) │ │
|
||||
│ │ - Policy Documents (Document Management) │ │
|
||||
│ │ - Versions (Versioning System) │ │
|
||||
│ │ - User Consent (Consent Tracking) │ │
|
||||
│ │ - Cookies (Cookie Categories & Vendors) │ │
|
||||
│ │ - Cookie Consent (Cookie Consent Tracking) │ │
|
||||
│ │ - DB Health Check (System Monitoring) │ │
|
||||
│ └───────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
│ ┌───────────────────────────────────────────────────┐ │
|
||||
│ │ Drizzle ORM Layer │ │
|
||||
│ ├───────────────────────────────────────────────────┤ │
|
||||
│ │ - Schema Definitions │ │
|
||||
│ │ - Relations │ │
|
||||
│ │ - Database Connection Pool │ │
|
||||
│ └───────────────────────────────────────────────────┘ │
|
||||
│ │ │
|
||||
└──────────────────────────┼────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ PostgreSQL │
|
||||
│ Database │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
### Frontend-Architektur
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ Angular Frontend │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||
│ │ Guards │ │ Interceptors │ │ Services │ │
|
||||
│ ├──────────────┤ ├──────────────┤ ├──────────────┤ │
|
||||
│ │ - AuthGuard │ │ - HTTP │ │ - Auth │ │
|
||||
│ │ │ │ - Error │ │ - REST API │ │
|
||||
│ │ │ │ │ │ - Session │ │
|
||||
│ │ │ │ │ │ - Security │ │
|
||||
│ └──────────────┘ └──────────────┘ └──────────────┘ │
|
||||
│ │
|
||||
│ ┌───────────────────────────────────────────────────┐ │
|
||||
│ │ Feature Modules │ │
|
||||
│ ├───────────────────────────────────────────────────┤ │
|
||||
│ │ ┌─────────────────────────────────────────┐ │ │
|
||||
│ │ │ Auth Module │ │ │
|
||||
│ │ │ - Login Component │ │ │
|
||||
│ │ └─────────────────────────────────────────┘ │ │
|
||||
│ │ │ │
|
||||
│ │ ┌─────────────────────────────────────────┐ │ │
|
||||
│ │ │ Project Dashboard │ │ │
|
||||
│ │ │ - Project List │ │ │
|
||||
│ │ │ - Project Creation │ │ │
|
||||
│ │ │ - Project Settings │ │ │
|
||||
│ │ └─────────────────────────────────────────┘ │ │
|
||||
│ │ │ │
|
||||
│ │ ┌─────────────────────────────────────────┐ │ │
|
||||
│ │ │ Individual Project Dashboard │ │ │
|
||||
│ │ │ - Agreements (Policy Documents) │ │ │
|
||||
│ │ │ - Cookie Consent Management │ │ │
|
||||
│ │ │ - FAQ Management │ │ │
|
||||
│ │ │ - Licenses Management │ │ │
|
||||
│ │ │ - User Management │ │ │
|
||||
│ │ │ - Project Settings │ │ │
|
||||
│ │ └─────────────────────────────────────────┘ │ │
|
||||
│ │ │ │
|
||||
│ │ ┌─────────────────────────────────────────┐ │ │
|
||||
│ │ │ Shared Components │ │ │
|
||||
│ │ │ - Settings │ │ │
|
||||
│ │ │ - Common UI Elements │ │ │
|
||||
│ │ └─────────────────────────────────────────┘ │ │
|
||||
│ └───────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
│ HTTPS/REST API
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ NestJS Backend │
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
### Datenbankbeziehungen
|
||||
|
||||
```
|
||||
┌──────────┐ ┌─────────────────┐ ┌─────────────┐
|
||||
│ Admin │◄───────►│ AdminProjects │◄───────►│ Project │
|
||||
└──────────┘ └─────────────────┘ └─────────────┘
|
||||
│
|
||||
│ 1:N
|
||||
┌────────────────────────────────────┤
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────────────┐ ┌──────────────────────────┐
|
||||
│ Policy Document │ │ Categories Metadata │
|
||||
└──────────────────────┘ └──────────────────────────┘
|
||||
│ │
|
||||
│ 1:N │ 1:N
|
||||
▼ ▼
|
||||
┌──────────────────────┐ ┌──────────────────────────┐
|
||||
│ Policy Document Meta │ │ Categories Language Data │
|
||||
└──────────────────────┘ └──────────────────────────┘
|
||||
│ │
|
||||
│ 1:N │ 1:N
|
||||
▼ ▼
|
||||
┌──────────────────────┐ ┌──────────────────────────┐
|
||||
│ Version Meta │ │ Vendor Meta │
|
||||
└──────────────────────┘ └──────────────────────────┘
|
||||
│ │
|
||||
│ 1:N │ 1:N
|
||||
▼ ├──────────┐
|
||||
┌──────────────────────┐ ▼ ▼
|
||||
│ User Consent │ ┌─────────────────┐ ┌────────────┐
|
||||
└──────────────────────┘ │ Vendor Language │ │Sub Service │
|
||||
└─────────────────┘ └────────────┘
|
||||
┌──────────────────────┐
|
||||
│ Cookie Consent │◄─── Project
|
||||
└──────────────────────┘
|
||||
|
||||
┌─────────────────────────┐
|
||||
│ Global Cookie Metadata │◄─── Project
|
||||
└─────────────────────────┘
|
||||
│
|
||||
│ 1:N
|
||||
▼
|
||||
┌─────────────────────────────┐
|
||||
│ Global Cookie Language Data │
|
||||
└─────────────────────────────────┘
|
||||
|
||||
┌──────────────────┐
|
||||
│ Project Keys │◄─── Project
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
### Sicherheitsarchitektur
|
||||
|
||||
#### Authentifizierung & Autorisierung
|
||||
1. **JWT-basierte Authentifizierung**
|
||||
- Access Token (kurzlebig)
|
||||
- Refresh Token (langlebig)
|
||||
- Token-Refresh-Mechanismus
|
||||
|
||||
2. **Rollenbasierte Zugriffskontrolle (RBAC)**
|
||||
- Super Admin (Role 1): Vollzugriff
|
||||
- Admin (Role 2): Projektbezogener Zugriff
|
||||
- Guard-basierte Absicherung auf Controller-Ebene
|
||||
|
||||
3. **Encryption-based Authentication**
|
||||
- Für externe/mobile Zugriffe
|
||||
- Token-basierte Verschlüsselung
|
||||
- User + Project ID Validierung
|
||||
|
||||
#### Security Features
|
||||
- **Rate Limiting**: Throttler mit konfigurierbaren Limits
|
||||
- **Password Security**: bcrypt Hashing mit Salt
|
||||
- **Account Lock**: Nach mehrfachen Fehlversuchen
|
||||
- **OTP-basierte Passwort-Wiederherstellung**
|
||||
- **Input Validation**: class-validator auf allen DTOs
|
||||
- **HTML Sanitization**: DOMPurify im Frontend
|
||||
- **CORS Configuration**: Custom Headers Middleware
|
||||
- **Soft Delete**: Keine permanente Löschung von Daten
|
||||
|
||||
---
|
||||
|
||||
## Deployment und Konfiguration
|
||||
|
||||
### Backend Environment Variables
|
||||
```env
|
||||
DATABASE_URL=postgresql://username:password@host:port/database
|
||||
NODE_ENV=development|test|production|local|demo
|
||||
PORT=3000
|
||||
JWT_SECRET=your_jwt_secret
|
||||
JWT_REFRESH_SECRET=your_refresh_secret
|
||||
ROOT_SECRET=your_root_secret
|
||||
ENCRYPTION_KEY=your_encryption_key
|
||||
SMTP_HOST=smtp.example.com
|
||||
SMTP_PORT=587
|
||||
SMTP_USER=your_email
|
||||
SMTP_PASSWORD=your_password
|
||||
```
|
||||
|
||||
### Frontend Environment
|
||||
```typescript
|
||||
{
|
||||
production: false,
|
||||
BASE_URL: "https://api.example.com/api/",
|
||||
TITLE: "Policy Vault - Environment"
|
||||
}
|
||||
```
|
||||
|
||||
### Datenbank-Setup
|
||||
```bash
|
||||
# Migrationen ausführen
|
||||
npm run migration:up
|
||||
|
||||
# Migrationen zurückrollen
|
||||
npm run migration:down
|
||||
|
||||
# Schema generieren
|
||||
npx drizzle-kit push
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## API-Sicherheit
|
||||
|
||||
### Token-basierte Authentifizierung
|
||||
- Alle geschützten Endpoints erfordern einen gültigen JWT-Token im Authorization-Header
|
||||
- Format: `Authorization: Bearer <access_token>`
|
||||
|
||||
### Encryption-based Endpoints
|
||||
Für mobile/externe Zugriffe (Consent Tracking):
|
||||
- Header: `secret` oder `mobiletoken`
|
||||
- Format: Verschlüsselter String mit `userId_projectId`
|
||||
- Automatische Validierung durch DecryptMiddleware
|
||||
|
||||
### Rate Limiting
|
||||
- Standard: 10 Requests pro Minute
|
||||
- OTP/Login: 3 Requests pro Minute
|
||||
- Konfigurierbar über ThrottlerModule
|
||||
|
||||
---
|
||||
|
||||
## Besondere Features
|
||||
|
||||
### 1. Versionierung
|
||||
- Komplettes Versions-Management für Policy Documents
|
||||
- Mehrsprachige Versionen mit separaten Inhalten
|
||||
- Publish/Draft Status
|
||||
- Historische Versionsverfolgung
|
||||
|
||||
### 2. Mehrsprachigkeit
|
||||
- Zentrale Sprach-Konfiguration pro Projekt
|
||||
- Separate Language-Data Tabellen für alle Inhaltstypen
|
||||
- Support für unbegrenzte Sprachen
|
||||
|
||||
### 3. Cookie-Consent-System
|
||||
- Granulare Kontrolle über Cookie-Kategorien
|
||||
- Vendor-Management mit Sub-Services
|
||||
- Plattform-spezifische Kategorien (Web, Mobile, etc.)
|
||||
- Versions-Tracking für Compliance
|
||||
|
||||
### 4. Rich Content Editing
|
||||
- CKEditor 5 Integration
|
||||
- Support für komplexe Formatierungen
|
||||
- Bild-Upload und -Verwaltung
|
||||
- Code-Block-Unterstützung
|
||||
|
||||
### 5. Logging & Monitoring
|
||||
- Winston-basiertes Logging
|
||||
- Daily Rotate Files
|
||||
- Structured Logging
|
||||
- Fehler-Tracking
|
||||
- Datenbank-Health-Checks
|
||||
|
||||
### 6. Soft Delete Pattern
|
||||
- Keine permanente Datenlöschung
|
||||
- `isDeleted` Flags auf allen Haupt-Entitäten
|
||||
- Möglichkeit zur Wiederherstellung
|
||||
- Audit Trail Erhaltung
|
||||
|
||||
---
|
||||
|
||||
## Entwicklung
|
||||
|
||||
### Backend starten
|
||||
```bash
|
||||
# Development
|
||||
npm run start:dev
|
||||
|
||||
# Local (mit Watch)
|
||||
npm run start:local
|
||||
|
||||
# Production
|
||||
npm run start:prod
|
||||
```
|
||||
|
||||
### Frontend starten
|
||||
```bash
|
||||
# Development Server
|
||||
npm run start
|
||||
# oder
|
||||
ng serve
|
||||
|
||||
# Build
|
||||
npm run build
|
||||
|
||||
# Mit PM2
|
||||
npm run start:pm2
|
||||
```
|
||||
|
||||
### Tests
|
||||
```bash
|
||||
# Backend Tests
|
||||
npm run test
|
||||
npm run test:e2e
|
||||
npm run test:cov
|
||||
|
||||
# Frontend Tests
|
||||
npm run test
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Zusammenfassung
|
||||
|
||||
Policy Vault ist eine umfassende Enterprise-Lösung für die Verwaltung von Datenschutzrichtlinien und Cookie-Einwilligungen. Das System bietet:
|
||||
|
||||
- **Multi-Tenant-Architektur** mit Projekt-basierter Trennung
|
||||
- **Robuste Authentifizierung** mit JWT und rollenbasierter Zugriffskontrolle
|
||||
- **Vollständiges Versions-Management** für Compliance-Tracking
|
||||
- **Granulare Cookie-Consent-Verwaltung** mit Vendor-Support
|
||||
- **Mehrsprachige Unterstützung** für globale Anwendungen
|
||||
- **Moderne Tech-Stack** mit NestJS, Angular und PostgreSQL
|
||||
- **Enterprise-Grade Security** mit Encryption, Rate Limiting und Audit Trails
|
||||
- **Skalierbare Architektur** mit klarer Trennung von Concerns
|
||||
|
||||
Das System eignet sich ideal für Unternehmen, die:
|
||||
- Multiple Projekte/Produkte mit unterschiedlichen Datenschutzrichtlinien verwalten
|
||||
- GDPR/DSGVO-Compliance sicherstellen müssen
|
||||
- Granulare Cookie-Einwilligungen tracken wollen
|
||||
- Mehrsprachige Anwendungen betreiben
|
||||
- Eine zentrale Policy-Management-Plattform benötigen
|
||||
1204
admin-v2/SBOM.md
Normal file
1204
admin-v2/SBOM.md
Normal file
File diff suppressed because it is too large
Load Diff
530
admin-v2/SOURCE_POLICY_IMPLEMENTATION_PLAN.md
Normal file
530
admin-v2/SOURCE_POLICY_IMPLEMENTATION_PLAN.md
Normal file
@@ -0,0 +1,530 @@
|
||||
# Source-Policy System - Implementierungsplan
|
||||
|
||||
## Zusammenfassung
|
||||
|
||||
Whitelist-basiertes Datenquellen-Management fuer das edu-search-service unter `/compliance/source-policy`. Fuer Auditoren pruefbar mit vollstaendigem Audit-Trail.
|
||||
|
||||
**Kernprinzipien:**
|
||||
- Nur offizielle Open-Data-Portale und amtliche Quellen (§5 UrhG)
|
||||
- Training mit externen Daten: **VERBOTEN**
|
||||
- Alle Aenderungen protokolliert (Audit-Trail)
|
||||
- PII-Blocklist mit Hard-Block
|
||||
|
||||
---
|
||||
|
||||
## 1. Architektur
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ admin-v2 (Next.js) │
|
||||
│ /app/(admin)/compliance/source-policy/ │
|
||||
│ ├── page.tsx (Dashboard + Tabs) │
|
||||
│ └── components/ │
|
||||
│ ├── SourcesTab.tsx (Whitelist-Verwaltung) │
|
||||
│ ├── OperationsMatrixTab.tsx (Lookup/RAG/Training/Export) │
|
||||
│ ├── PIIRulesTab.tsx (PII-Blocklist) │
|
||||
│ └── AuditTab.tsx (Aenderungshistorie + Export) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ edu-search-service (Go) │
|
||||
│ NEW: internal/policy/ │
|
||||
│ ├── models.go (Datenstrukturen) │
|
||||
│ ├── store.go (PostgreSQL CRUD) │
|
||||
│ ├── enforcer.go (Policy-Enforcement) │
|
||||
│ ├── pii_detector.go (PII-Erkennung) │
|
||||
│ └── audit.go (Audit-Logging) │
|
||||
│ │
|
||||
│ MODIFIED: │
|
||||
│ ├── crawler/crawler.go (Whitelist-Check vor Fetch) │
|
||||
│ ├── pipeline/pipeline.go (PII-Filter nach Extract) │
|
||||
│ └── api/handlers/policy_handlers.go (Admin-API) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ PostgreSQL │
|
||||
│ NEW TABLES: │
|
||||
│ - source_policies (versionierte Policies) │
|
||||
│ - allowed_sources (Whitelist pro Bundesland) │
|
||||
│ - operation_permissions (Lookup/RAG/Training/Export Matrix) │
|
||||
│ - pii_rules (Regex/Keyword Blocklist) │
|
||||
│ - policy_audit_log (unveraenderlich) │
|
||||
│ - blocked_content_log (blockierte URLs fuer Audit) │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Datenmodell
|
||||
|
||||
### 2.1 PostgreSQL Schema
|
||||
|
||||
```sql
|
||||
-- Policies (versioniert)
|
||||
CREATE TABLE source_policies (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
version INTEGER NOT NULL DEFAULT 1,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
bundesland VARCHAR(2), -- NULL = Bundesebene/KMK
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
approved_by UUID,
|
||||
approved_at TIMESTAMP
|
||||
);
|
||||
|
||||
-- Whitelist
|
||||
CREATE TABLE allowed_sources (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
policy_id UUID REFERENCES source_policies(id),
|
||||
domain VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
license VARCHAR(50) NOT NULL, -- DL-DE-BY-2.0, CC-BY, §5 UrhG
|
||||
legal_basis VARCHAR(100),
|
||||
citation_template TEXT,
|
||||
trust_boost DECIMAL(3,2) DEFAULT 0.50,
|
||||
is_active BOOLEAN DEFAULT true
|
||||
);
|
||||
|
||||
-- Operations Matrix
|
||||
CREATE TABLE operation_permissions (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
source_id UUID REFERENCES allowed_sources(id),
|
||||
operation VARCHAR(50) NOT NULL, -- lookup, rag, training, export
|
||||
is_allowed BOOLEAN NOT NULL,
|
||||
requires_citation BOOLEAN DEFAULT false,
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
-- PII Blocklist
|
||||
CREATE TABLE pii_rules (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
rule_type VARCHAR(50) NOT NULL, -- regex, keyword
|
||||
pattern TEXT NOT NULL,
|
||||
severity VARCHAR(20) DEFAULT 'block', -- block, warn, redact
|
||||
is_active BOOLEAN DEFAULT true
|
||||
);
|
||||
|
||||
-- Audit Log (immutable)
|
||||
CREATE TABLE policy_audit_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
action VARCHAR(50) NOT NULL,
|
||||
entity_type VARCHAR(50) NOT NULL,
|
||||
entity_id UUID,
|
||||
old_value JSONB,
|
||||
new_value JSONB,
|
||||
user_email VARCHAR(255),
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Blocked Content Log
|
||||
CREATE TABLE blocked_content_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
url VARCHAR(2048) NOT NULL,
|
||||
domain VARCHAR(255) NOT NULL,
|
||||
block_reason VARCHAR(100) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
```
|
||||
|
||||
### 2.2 Initial-Daten
|
||||
|
||||
Datei: `edu-search-service/policies/bundeslaender.yaml`
|
||||
|
||||
```yaml
|
||||
federal:
|
||||
name: "KMK & Bundesebene"
|
||||
sources:
|
||||
- domain: "kmk.org"
|
||||
name: "Kultusministerkonferenz"
|
||||
license: "§5 UrhG"
|
||||
legal_basis: "Amtliche Werke (§5 UrhG)"
|
||||
citation_template: "Quelle: KMK, {title}, {date}"
|
||||
- domain: "bildungsserver.de"
|
||||
name: "Deutscher Bildungsserver"
|
||||
license: "DL-DE-BY-2.0"
|
||||
|
||||
NI:
|
||||
name: "Niedersachsen"
|
||||
sources:
|
||||
- domain: "nibis.de"
|
||||
name: "NiBiS Bildungsserver"
|
||||
license: "DL-DE-BY-2.0"
|
||||
- domain: "mk.niedersachsen.de"
|
||||
name: "Kultusministerium Niedersachsen"
|
||||
license: "§5 UrhG"
|
||||
- domain: "cuvo.nibis.de"
|
||||
name: "Kerncurricula Niedersachsen"
|
||||
license: "DL-DE-BY-2.0"
|
||||
|
||||
BY:
|
||||
name: "Bayern"
|
||||
sources:
|
||||
- domain: "km.bayern.de"
|
||||
name: "Bayerisches Kultusministerium"
|
||||
license: "§5 UrhG"
|
||||
- domain: "isb.bayern.de"
|
||||
name: "ISB Bayern"
|
||||
license: "DL-DE-BY-2.0"
|
||||
- domain: "lehrplanplus.bayern.de"
|
||||
name: "LehrplanPLUS"
|
||||
license: "DL-DE-BY-2.0"
|
||||
|
||||
# Default Operations Matrix
|
||||
default_operations:
|
||||
lookup:
|
||||
allowed: true
|
||||
requires_citation: true
|
||||
rag:
|
||||
allowed: true
|
||||
requires_citation: true
|
||||
training:
|
||||
allowed: false # VERBOTEN
|
||||
export:
|
||||
allowed: true
|
||||
requires_citation: true
|
||||
|
||||
# Default PII Rules
|
||||
pii_rules:
|
||||
- name: "Email Addresses"
|
||||
type: "regex"
|
||||
pattern: "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"
|
||||
severity: "block"
|
||||
- name: "German Phone Numbers"
|
||||
type: "regex"
|
||||
pattern: "(?:\\+49|0)[\\s.-]?\\d{2,4}[\\s.-]?\\d{3,}[\\s.-]?\\d{2,}"
|
||||
severity: "block"
|
||||
- name: "IBAN"
|
||||
type: "regex"
|
||||
pattern: "DE\\d{2}\\s?\\d{4}\\s?\\d{4}\\s?\\d{4}\\s?\\d{4}\\s?\\d{2}"
|
||||
severity: "block"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Backend Implementation
|
||||
|
||||
### 3.1 Neue Dateien
|
||||
|
||||
| Datei | Beschreibung |
|
||||
|-------|--------------|
|
||||
| `internal/policy/models.go` | Go Structs (SourcePolicy, AllowedSource, PIIRule, etc.) |
|
||||
| `internal/policy/store.go` | PostgreSQL CRUD mit pgx |
|
||||
| `internal/policy/enforcer.go` | `CheckSource()`, `CheckOperation()`, `DetectPII()` |
|
||||
| `internal/policy/audit.go` | `LogChange()`, `LogBlocked()` |
|
||||
| `internal/policy/pii_detector.go` | Regex-basierte PII-Erkennung |
|
||||
| `internal/api/handlers/policy_handlers.go` | Admin-Endpoints |
|
||||
| `migrations/005_source_policies.sql` | DB-Schema |
|
||||
| `policies/bundeslaender.yaml` | Initial-Daten |
|
||||
|
||||
### 3.2 API Endpoints
|
||||
|
||||
```
|
||||
# Policies
|
||||
GET /v1/admin/policies
|
||||
POST /v1/admin/policies
|
||||
PUT /v1/admin/policies/:id
|
||||
|
||||
# Sources (Whitelist)
|
||||
GET /v1/admin/sources
|
||||
POST /v1/admin/sources
|
||||
PUT /v1/admin/sources/:id
|
||||
DELETE /v1/admin/sources/:id
|
||||
|
||||
# Operations Matrix
|
||||
GET /v1/admin/operations-matrix
|
||||
PUT /v1/admin/operations/:id
|
||||
|
||||
# PII Rules
|
||||
GET /v1/admin/pii-rules
|
||||
POST /v1/admin/pii-rules
|
||||
PUT /v1/admin/pii-rules/:id
|
||||
DELETE /v1/admin/pii-rules/:id
|
||||
POST /v1/admin/pii-rules/test # Test gegen Sample-Text
|
||||
|
||||
# Audit
|
||||
GET /v1/admin/policy-audit?from=&to=
|
||||
GET /v1/admin/blocked-content?from=&to=
|
||||
GET /v1/admin/compliance-report # PDF/JSON Export
|
||||
|
||||
# Live-Check
|
||||
POST /v1/admin/check-compliance
|
||||
Body: { "url": "...", "operation": "lookup" }
|
||||
```
|
||||
|
||||
### 3.3 Crawler-Integration
|
||||
|
||||
In `crawler/crawler.go`:
|
||||
```go
|
||||
func (c *Crawler) FetchWithPolicy(ctx context.Context, url string) (*FetchResult, error) {
|
||||
// 1. Whitelist-Check
|
||||
source, err := c.enforcer.CheckSource(ctx, url)
|
||||
if err != nil || source == nil {
|
||||
c.enforcer.LogBlocked(ctx, url, "not_whitelisted")
|
||||
return nil, ErrNotWhitelisted
|
||||
}
|
||||
|
||||
// ... existing fetch ...
|
||||
|
||||
// 2. PII-Check nach Fetch
|
||||
piiMatches := c.enforcer.DetectPII(content)
|
||||
if hasSeverity(piiMatches, "block") {
|
||||
c.enforcer.LogBlocked(ctx, url, "pii_detected")
|
||||
return nil, ErrPIIDetected
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Frontend Implementation
|
||||
|
||||
### 4.1 Navigation Update
|
||||
|
||||
In `lib/navigation.ts` unter `compliance` Kategorie hinzufuegen:
|
||||
|
||||
```typescript
|
||||
{
|
||||
id: 'source-policy',
|
||||
name: 'Quellen-Policy',
|
||||
href: '/compliance/source-policy',
|
||||
description: 'Datenquellen & Compliance',
|
||||
purpose: 'Whitelist zugelassener Datenquellen mit Operations-Matrix und PII-Blocklist.',
|
||||
audience: ['DSB', 'Compliance Officer', 'Auditor'],
|
||||
gdprArticles: ['Art. 5 (Rechtmaessigkeit)', 'Art. 6 (Rechtsgrundlage)'],
|
||||
}
|
||||
```
|
||||
|
||||
### 4.2 Seiten-Struktur
|
||||
|
||||
```
|
||||
/app/(admin)/compliance/source-policy/
|
||||
├── page.tsx # Haupt-Dashboard mit Tabs
|
||||
└── components/
|
||||
├── SourcesTab.tsx # Whitelist-Tabelle mit CRUD
|
||||
├── OperationsMatrixTab.tsx # 4x4 Matrix
|
||||
├── PIIRulesTab.tsx # PII-Regeln mit Test-Funktion
|
||||
└── AuditTab.tsx # Aenderungshistorie + Export
|
||||
```
|
||||
|
||||
### 4.3 UI-Layout
|
||||
|
||||
**Stats Cards (oben):**
|
||||
- Aktive Policies
|
||||
- Zugelassene Quellen
|
||||
- Blockiert (heute)
|
||||
- Compliance Score
|
||||
|
||||
**Tabs:**
|
||||
1. **Dashboard** - Uebersicht mit Quick-Stats
|
||||
2. **Quellen** - Whitelist-Tabelle (Domain, Name, Lizenz, Status)
|
||||
3. **Operations** - Matrix mit Lookup/RAG/Training/Export
|
||||
4. **PII-Regeln** - Blocklist mit Test-Funktion
|
||||
5. **Audit** - Aenderungshistorie mit PDF/JSON-Export
|
||||
|
||||
**Pattern (aus audit-report/page.tsx):**
|
||||
- Tab-Navigation: `bg-purple-600 text-white` fuer aktiv
|
||||
- Status-Badges: `bg-green-100 text-green-700` fuer aktiv
|
||||
- Tabellen: `hover:bg-slate-50`
|
||||
- Info-Boxen: `bg-blue-50 border-blue-200`
|
||||
|
||||
---
|
||||
|
||||
## 5. Betroffene Dateien
|
||||
|
||||
### Neue Dateien erstellen:
|
||||
|
||||
**Backend (edu-search-service):**
|
||||
```
|
||||
internal/policy/models.go
|
||||
internal/policy/store.go
|
||||
internal/policy/enforcer.go
|
||||
internal/policy/audit.go
|
||||
internal/policy/pii_detector.go
|
||||
internal/api/handlers/policy_handlers.go
|
||||
migrations/005_source_policies.sql
|
||||
policies/bundeslaender.yaml
|
||||
```
|
||||
|
||||
**Frontend (admin-v2):**
|
||||
```
|
||||
app/(admin)/compliance/source-policy/page.tsx
|
||||
app/(admin)/compliance/source-policy/components/SourcesTab.tsx
|
||||
app/(admin)/compliance/source-policy/components/OperationsMatrixTab.tsx
|
||||
app/(admin)/compliance/source-policy/components/PIIRulesTab.tsx
|
||||
app/(admin)/compliance/source-policy/components/AuditTab.tsx
|
||||
```
|
||||
|
||||
### Bestehende Dateien aendern:
|
||||
|
||||
```
|
||||
edu-search-service/cmd/server/main.go # Policy-Endpoints registrieren
|
||||
edu-search-service/internal/crawler/crawler.go # Policy-Check hinzufuegen
|
||||
edu-search-service/internal/pipeline/pipeline.go # PII-Filter
|
||||
edu-search-service/internal/database/database.go # Migrations
|
||||
admin-v2/lib/navigation.ts # source-policy Modul
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Implementierungs-Reihenfolge
|
||||
|
||||
### Phase 1: Datenbank & Models
|
||||
1. Migration `005_source_policies.sql` erstellen
|
||||
2. Go Models in `internal/policy/models.go`
|
||||
3. Store-Layer in `internal/policy/store.go`
|
||||
4. YAML-Loader fuer Initial-Daten
|
||||
|
||||
### Phase 2: Policy Enforcer
|
||||
1. `internal/policy/enforcer.go` - CheckSource, CheckOperation
|
||||
2. `internal/policy/pii_detector.go` - Regex-basierte Erkennung
|
||||
3. `internal/policy/audit.go` - Logging
|
||||
4. Integration in Crawler
|
||||
|
||||
### Phase 3: Admin API
|
||||
1. `internal/api/handlers/policy_handlers.go`
|
||||
2. Routen in main.go registrieren
|
||||
3. API testen
|
||||
|
||||
### Phase 4: Frontend
|
||||
1. Hauptseite mit PagePurpose
|
||||
2. SourcesTab mit Whitelist-CRUD
|
||||
3. OperationsMatrixTab
|
||||
4. PIIRulesTab mit Test-Funktion
|
||||
5. AuditTab mit Export
|
||||
|
||||
### Phase 5: Testing & Deployment
|
||||
1. Unit Tests fuer Enforcer
|
||||
2. Integration Tests fuer API
|
||||
3. E2E Test fuer Frontend
|
||||
4. Deployment auf Mac Mini
|
||||
|
||||
---
|
||||
|
||||
## 7. Verifikation
|
||||
|
||||
### Nach Backend (Phase 1-3):
|
||||
```bash
|
||||
# Migration ausfuehren
|
||||
ssh macmini "cd /path/to/edu-search-service && go run ./cmd/migrate"
|
||||
|
||||
# API testen
|
||||
curl -X GET http://macmini:8088/v1/admin/policies
|
||||
curl -X POST http://macmini:8088/v1/admin/check-compliance \
|
||||
-d '{"url":"https://nibis.de/test","operation":"lookup"}'
|
||||
```
|
||||
|
||||
### Nach Frontend (Phase 4):
|
||||
```bash
|
||||
# Build & Deploy
|
||||
rsync -avz admin-v2/ macmini:/path/to/admin-v2/
|
||||
ssh macmini "docker compose build admin-v2 && docker compose up -d admin-v2"
|
||||
|
||||
# Testen
|
||||
open https://macmini:3002/compliance/source-policy
|
||||
```
|
||||
|
||||
### Auditor-Checkliste:
|
||||
- [ ] Alle Quellen in Whitelist dokumentiert
|
||||
- [ ] Operations-Matrix zeigt Training = VERBOTEN
|
||||
- [ ] PII-Regeln aktiv und testbar
|
||||
- [ ] Audit-Log zeigt alle Aenderungen
|
||||
- [ ] Blocked-Content-Log zeigt blockierte URLs
|
||||
- [ ] PDF/JSON-Export funktioniert
|
||||
|
||||
---
|
||||
|
||||
## 8. KMK-Spezifika (§5 UrhG)
|
||||
|
||||
**Rechtsgrundlage:**
|
||||
- KMK-Beschluesse, Vereinbarungen, EPA sind amtliche Werke nach §5 UrhG
|
||||
- Frei nutzbar, Attribution erforderlich
|
||||
|
||||
**Zitierformat:**
|
||||
```
|
||||
Quelle: KMK, [Titel des Beschlusses], [Datum]
|
||||
Beispiel: Quelle: KMK, Bildungsstandards im Fach Deutsch, 2003
|
||||
```
|
||||
|
||||
**Zugelassene Dokumenttypen:**
|
||||
- Beschluesse (Resolutions)
|
||||
- Vereinbarungen (Agreements)
|
||||
- EPA (Einheitliche Pruefungsanforderungen)
|
||||
- Empfehlungen (Recommendations)
|
||||
|
||||
**In Operations-Matrix:**
|
||||
| Operation | Erlaubt | Hinweis |
|
||||
|-----------|---------|---------|
|
||||
| Lookup | Ja | Quelle anzeigen |
|
||||
| RAG | Ja | Zitation im Output |
|
||||
| Training | **NEIN** | VERBOTEN |
|
||||
| Export | Ja | Attribution |
|
||||
|
||||
---
|
||||
|
||||
## 9. Lizenzen
|
||||
|
||||
| Lizenz | Name | Attribution |
|
||||
|--------|------|-------------|
|
||||
| DL-DE-BY-2.0 | Datenlizenz Deutschland | Ja |
|
||||
| CC-BY | Creative Commons Attribution | Ja |
|
||||
| CC-BY-SA | CC Attribution-ShareAlike | Ja + ShareAlike |
|
||||
| CC0 | Public Domain | Nein |
|
||||
| §5 UrhG | Amtliche Werke | Ja (Quelle) |
|
||||
|
||||
---
|
||||
|
||||
## 10. Aktueller Stand
|
||||
|
||||
**Phase 1: Datenbank & Models - ABGESCHLOSSEN**
|
||||
- [x] Codebase-Exploration edu-search-service
|
||||
- [x] Codebase-Exploration admin-v2
|
||||
- [x] Plan dokumentiert
|
||||
- [x] Migration 005_source_policies.sql erstellen
|
||||
- [x] Go Models implementieren (internal/policy/models.go)
|
||||
- [x] Store-Layer implementieren (internal/policy/store.go)
|
||||
- [x] Policy Enforcer implementieren (internal/policy/enforcer.go)
|
||||
- [x] PII Detector implementieren (internal/policy/pii_detector.go)
|
||||
- [x] Audit Logging implementieren (internal/policy/audit.go)
|
||||
- [x] YAML Loader implementieren (internal/policy/loader.go)
|
||||
- [x] Initial-Daten YAML erstellen (policies/bundeslaender.yaml)
|
||||
- [x] Unit Tests schreiben (internal/policy/policy_test.go)
|
||||
- [x] README aktualisieren
|
||||
|
||||
**Phase 2: Admin API - AUSSTEHEND**
|
||||
- [ ] API Handlers implementieren (policy_handlers.go)
|
||||
- [ ] main.go aktualisieren
|
||||
- [ ] API testen
|
||||
|
||||
**Phase 3: Integration - AUSSTEHEND**
|
||||
- [ ] Crawler-Integration
|
||||
- [ ] Pipeline-Integration
|
||||
|
||||
**Phase 4: Frontend - AUSSTEHEND**
|
||||
- [ ] Frontend page.tsx erstellen
|
||||
- [ ] SourcesTab Component
|
||||
- [ ] OperationsMatrixTab Component
|
||||
- [ ] PIIRulesTab Component
|
||||
- [ ] AuditTab Component
|
||||
- [ ] Navigation aktualisieren
|
||||
|
||||
**Erstellte Dateien:**
|
||||
```
|
||||
edu-search-service/
|
||||
├── migrations/
|
||||
│ └── 005_source_policies.sql # DB Schema (6 Tabellen)
|
||||
├── internal/policy/
|
||||
│ ├── models.go # Datenstrukturen & Enums
|
||||
│ ├── store.go # PostgreSQL CRUD
|
||||
│ ├── enforcer.go # Policy-Enforcement
|
||||
│ ├── pii_detector.go # PII-Erkennung
|
||||
│ ├── audit.go # Audit-Logging
|
||||
│ ├── loader.go # YAML-Loader
|
||||
│ └── policy_test.go # Unit Tests
|
||||
└── policies/
|
||||
└── bundeslaender.yaml # Initial-Daten (8 Bundeslaender)
|
||||
```
|
||||
@@ -0,0 +1,305 @@
|
||||
-- Migration: Create Academy Tables
|
||||
-- Description: Schema for the Compliance Academy module (courses, lessons, quizzes, enrollments, certificates, progress)
|
||||
|
||||
-- Enable UUID extension if not already enabled
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
-- ============================================================================
|
||||
-- 1. academy_courses - Training courses for compliance education
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS academy_courses (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
tenant_id VARCHAR(255) NOT NULL,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
category VARCHAR(50),
|
||||
passing_score INTEGER DEFAULT 70,
|
||||
duration_minutes INTEGER,
|
||||
required_for_roles JSONB DEFAULT '["all"]',
|
||||
status VARCHAR(50) DEFAULT 'draft',
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes for academy_courses
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_courses_tenant ON academy_courses(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_courses_status ON academy_courses(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_courses_category ON academy_courses(category);
|
||||
|
||||
-- Auto-update trigger for academy_courses.updated_at
|
||||
CREATE OR REPLACE FUNCTION update_academy_courses_updated_at()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trigger_academy_courses_updated_at ON academy_courses;
|
||||
CREATE TRIGGER trigger_academy_courses_updated_at
|
||||
BEFORE UPDATE ON academy_courses
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_academy_courses_updated_at();
|
||||
|
||||
-- Comments for academy_courses
|
||||
COMMENT ON TABLE academy_courses IS 'Stores compliance training courses per tenant';
|
||||
COMMENT ON COLUMN academy_courses.tenant_id IS 'Identifier for the tenant owning this course';
|
||||
COMMENT ON COLUMN academy_courses.title IS 'Course title displayed to users';
|
||||
COMMENT ON COLUMN academy_courses.category IS 'Course category (e.g. dsgvo, ai-act, security)';
|
||||
COMMENT ON COLUMN academy_courses.passing_score IS 'Minimum score (0-100) required to pass the course';
|
||||
COMMENT ON COLUMN academy_courses.duration_minutes IS 'Estimated total duration of the course in minutes';
|
||||
COMMENT ON COLUMN academy_courses.required_for_roles IS 'JSON array of roles required to complete this course';
|
||||
COMMENT ON COLUMN academy_courses.status IS 'Course status: draft, published, archived';
|
||||
|
||||
-- ============================================================================
|
||||
-- 2. academy_lessons - Individual lessons within a course
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS academy_lessons (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
course_id UUID NOT NULL REFERENCES academy_courses(id) ON DELETE CASCADE,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
type VARCHAR(20) NOT NULL,
|
||||
content_markdown TEXT,
|
||||
video_url VARCHAR(500),
|
||||
audio_url VARCHAR(500),
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
duration_minutes INTEGER,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes for academy_lessons
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_lessons_course ON academy_lessons(course_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_lessons_sort ON academy_lessons(course_id, sort_order);
|
||||
|
||||
-- Auto-update trigger for academy_lessons.updated_at
|
||||
CREATE OR REPLACE FUNCTION update_academy_lessons_updated_at()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trigger_academy_lessons_updated_at ON academy_lessons;
|
||||
CREATE TRIGGER trigger_academy_lessons_updated_at
|
||||
BEFORE UPDATE ON academy_lessons
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_academy_lessons_updated_at();
|
||||
|
||||
-- Comments for academy_lessons
|
||||
COMMENT ON TABLE academy_lessons IS 'Individual lessons belonging to a course';
|
||||
COMMENT ON COLUMN academy_lessons.course_id IS 'Foreign key to the parent course';
|
||||
COMMENT ON COLUMN academy_lessons.type IS 'Lesson type: text, video, audio, quiz, interactive';
|
||||
COMMENT ON COLUMN academy_lessons.content_markdown IS 'Lesson content in Markdown format';
|
||||
COMMENT ON COLUMN academy_lessons.video_url IS 'URL to video content (if type is video)';
|
||||
COMMENT ON COLUMN academy_lessons.audio_url IS 'URL to audio content (if type is audio)';
|
||||
COMMENT ON COLUMN academy_lessons.sort_order IS 'Order of the lesson within the course';
|
||||
COMMENT ON COLUMN academy_lessons.duration_minutes IS 'Estimated duration of this lesson in minutes';
|
||||
|
||||
-- ============================================================================
|
||||
-- 3. academy_quiz_questions - Quiz questions attached to lessons
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS academy_quiz_questions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
lesson_id UUID NOT NULL REFERENCES academy_lessons(id) ON DELETE CASCADE,
|
||||
question TEXT NOT NULL,
|
||||
options JSONB NOT NULL,
|
||||
correct_option_index INTEGER NOT NULL,
|
||||
explanation TEXT,
|
||||
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes for academy_quiz_questions
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_quiz_questions_lesson ON academy_quiz_questions(lesson_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_quiz_questions_sort ON academy_quiz_questions(lesson_id, sort_order);
|
||||
|
||||
-- Comments for academy_quiz_questions
|
||||
COMMENT ON TABLE academy_quiz_questions IS 'Quiz questions belonging to a lesson';
|
||||
COMMENT ON COLUMN academy_quiz_questions.lesson_id IS 'Foreign key to the parent lesson';
|
||||
COMMENT ON COLUMN academy_quiz_questions.question IS 'The question text';
|
||||
COMMENT ON COLUMN academy_quiz_questions.options IS 'JSON array of answer options (strings)';
|
||||
COMMENT ON COLUMN academy_quiz_questions.correct_option_index IS 'Zero-based index of the correct option';
|
||||
COMMENT ON COLUMN academy_quiz_questions.explanation IS 'Explanation shown after answering (correct or incorrect)';
|
||||
COMMENT ON COLUMN academy_quiz_questions.sort_order IS 'Order of the question within the lesson quiz';
|
||||
|
||||
-- ============================================================================
|
||||
-- 4. academy_enrollments - User enrollments in courses
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS academy_enrollments (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
tenant_id VARCHAR(255) NOT NULL,
|
||||
course_id UUID NOT NULL REFERENCES academy_courses(id) ON DELETE CASCADE,
|
||||
user_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255),
|
||||
user_email VARCHAR(255),
|
||||
status VARCHAR(20) DEFAULT 'not_started',
|
||||
progress INTEGER DEFAULT 0,
|
||||
started_at TIMESTAMP WITH TIME ZONE,
|
||||
completed_at TIMESTAMP WITH TIME ZONE,
|
||||
certificate_id UUID,
|
||||
deadline TIMESTAMP WITH TIME ZONE,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes for academy_enrollments
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_enrollments_tenant ON academy_enrollments(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_enrollments_course ON academy_enrollments(course_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_enrollments_user ON academy_enrollments(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_enrollments_status ON academy_enrollments(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_enrollments_tenant_user ON academy_enrollments(tenant_id, user_id);
|
||||
|
||||
-- Auto-update trigger for academy_enrollments.updated_at
|
||||
CREATE OR REPLACE FUNCTION update_academy_enrollments_updated_at()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trigger_academy_enrollments_updated_at ON academy_enrollments;
|
||||
CREATE TRIGGER trigger_academy_enrollments_updated_at
|
||||
BEFORE UPDATE ON academy_enrollments
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_academy_enrollments_updated_at();
|
||||
|
||||
-- Comments for academy_enrollments
|
||||
COMMENT ON TABLE academy_enrollments IS 'Tracks user enrollments and progress in courses';
|
||||
COMMENT ON COLUMN academy_enrollments.tenant_id IS 'Identifier for the tenant';
|
||||
COMMENT ON COLUMN academy_enrollments.course_id IS 'Foreign key to the enrolled course';
|
||||
COMMENT ON COLUMN academy_enrollments.user_id IS 'Identifier of the enrolled user';
|
||||
COMMENT ON COLUMN academy_enrollments.user_name IS 'Display name of the enrolled user';
|
||||
COMMENT ON COLUMN academy_enrollments.user_email IS 'Email address of the enrolled user';
|
||||
COMMENT ON COLUMN academy_enrollments.status IS 'Enrollment status: not_started, in_progress, completed, expired';
|
||||
COMMENT ON COLUMN academy_enrollments.progress IS 'Completion percentage (0-100)';
|
||||
COMMENT ON COLUMN academy_enrollments.certificate_id IS 'Reference to issued certificate (if completed)';
|
||||
COMMENT ON COLUMN academy_enrollments.deadline IS 'Deadline by which the course must be completed';
|
||||
|
||||
-- ============================================================================
|
||||
-- 5. academy_certificates - Certificates issued upon course completion
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS academy_certificates (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
tenant_id VARCHAR(255) NOT NULL,
|
||||
enrollment_id UUID NOT NULL UNIQUE REFERENCES academy_enrollments(id) ON DELETE CASCADE,
|
||||
course_id UUID NOT NULL REFERENCES academy_courses(id) ON DELETE CASCADE,
|
||||
user_id VARCHAR(255) NOT NULL,
|
||||
user_name VARCHAR(255),
|
||||
course_name VARCHAR(255),
|
||||
score INTEGER,
|
||||
issued_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
|
||||
valid_until TIMESTAMP WITH TIME ZONE,
|
||||
pdf_url VARCHAR(500)
|
||||
);
|
||||
|
||||
-- Indexes for academy_certificates
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_certificates_tenant ON academy_certificates(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_certificates_user ON academy_certificates(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_certificates_course ON academy_certificates(course_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_certificates_enrollment ON academy_certificates(enrollment_id);
|
||||
|
||||
-- Comments for academy_certificates
|
||||
COMMENT ON TABLE academy_certificates IS 'Certificates issued when a user completes a course';
|
||||
COMMENT ON COLUMN academy_certificates.tenant_id IS 'Identifier for the tenant';
|
||||
COMMENT ON COLUMN academy_certificates.enrollment_id IS 'Unique reference to the enrollment (one certificate per enrollment)';
|
||||
COMMENT ON COLUMN academy_certificates.course_id IS 'Foreign key to the completed course';
|
||||
COMMENT ON COLUMN academy_certificates.user_id IS 'Identifier of the certified user';
|
||||
COMMENT ON COLUMN academy_certificates.user_name IS 'Name of the user as printed on the certificate';
|
||||
COMMENT ON COLUMN academy_certificates.course_name IS 'Name of the course as printed on the certificate';
|
||||
COMMENT ON COLUMN academy_certificates.score IS 'Final quiz score achieved (0-100)';
|
||||
COMMENT ON COLUMN academy_certificates.issued_at IS 'Timestamp when the certificate was issued';
|
||||
COMMENT ON COLUMN academy_certificates.valid_until IS 'Expiry date of the certificate (NULL = no expiry)';
|
||||
COMMENT ON COLUMN academy_certificates.pdf_url IS 'URL to the generated certificate PDF';
|
||||
|
||||
-- ============================================================================
|
||||
-- 6. academy_lesson_progress - Per-lesson progress tracking
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS academy_lesson_progress (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
enrollment_id UUID NOT NULL REFERENCES academy_enrollments(id) ON DELETE CASCADE,
|
||||
lesson_id UUID NOT NULL REFERENCES academy_lessons(id) ON DELETE CASCADE,
|
||||
completed BOOLEAN DEFAULT false,
|
||||
quiz_score INTEGER,
|
||||
completed_at TIMESTAMP WITH TIME ZONE,
|
||||
CONSTRAINT uq_academy_lesson_progress_enrollment_lesson UNIQUE (enrollment_id, lesson_id)
|
||||
);
|
||||
|
||||
-- Indexes for academy_lesson_progress
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_lesson_progress_enrollment ON academy_lesson_progress(enrollment_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_academy_lesson_progress_lesson ON academy_lesson_progress(lesson_id);
|
||||
|
||||
-- Comments for academy_lesson_progress
|
||||
COMMENT ON TABLE academy_lesson_progress IS 'Tracks completion status and quiz scores per lesson per enrollment';
|
||||
COMMENT ON COLUMN academy_lesson_progress.enrollment_id IS 'Foreign key to the enrollment';
|
||||
COMMENT ON COLUMN academy_lesson_progress.lesson_id IS 'Foreign key to the lesson';
|
||||
COMMENT ON COLUMN academy_lesson_progress.completed IS 'Whether the lesson has been completed';
|
||||
COMMENT ON COLUMN academy_lesson_progress.quiz_score IS 'Quiz score for this lesson (0-100), NULL if no quiz';
|
||||
COMMENT ON COLUMN academy_lesson_progress.completed_at IS 'Timestamp when the lesson was completed';
|
||||
|
||||
-- ============================================================================
|
||||
-- Helper: Upsert function for lesson progress (ON CONFLICT handling)
|
||||
-- ============================================================================
|
||||
|
||||
CREATE OR REPLACE FUNCTION upsert_academy_lesson_progress(
|
||||
p_enrollment_id UUID,
|
||||
p_lesson_id UUID,
|
||||
p_completed BOOLEAN,
|
||||
p_quiz_score INTEGER DEFAULT NULL
|
||||
)
|
||||
RETURNS academy_lesson_progress AS $$
|
||||
DECLARE
|
||||
result academy_lesson_progress;
|
||||
BEGIN
|
||||
INSERT INTO academy_lesson_progress (enrollment_id, lesson_id, completed, quiz_score, completed_at)
|
||||
VALUES (
|
||||
p_enrollment_id,
|
||||
p_lesson_id,
|
||||
p_completed,
|
||||
p_quiz_score,
|
||||
CASE WHEN p_completed THEN NOW() ELSE NULL END
|
||||
)
|
||||
ON CONFLICT ON CONSTRAINT uq_academy_lesson_progress_enrollment_lesson
|
||||
DO UPDATE SET
|
||||
completed = EXCLUDED.completed,
|
||||
quiz_score = COALESCE(EXCLUDED.quiz_score, academy_lesson_progress.quiz_score),
|
||||
completed_at = CASE
|
||||
WHEN EXCLUDED.completed AND academy_lesson_progress.completed_at IS NULL THEN NOW()
|
||||
WHEN NOT EXCLUDED.completed THEN NULL
|
||||
ELSE academy_lesson_progress.completed_at
|
||||
END
|
||||
RETURNING * INTO result;
|
||||
|
||||
RETURN result;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
COMMENT ON FUNCTION upsert_academy_lesson_progress IS 'Insert or update lesson progress with ON CONFLICT handling on the unique (enrollment_id, lesson_id) constraint';
|
||||
|
||||
-- ============================================================================
|
||||
-- Helper: Cleanup function for expired certificates
|
||||
-- ============================================================================
|
||||
|
||||
CREATE OR REPLACE FUNCTION cleanup_expired_academy_certificates(days_past_expiry INTEGER DEFAULT 0)
|
||||
RETURNS INTEGER AS $$
|
||||
DECLARE
|
||||
deleted_count INTEGER;
|
||||
BEGIN
|
||||
DELETE FROM academy_certificates
|
||||
WHERE valid_until IS NOT NULL
|
||||
AND valid_until < NOW() - (days_past_expiry || ' days')::INTERVAL;
|
||||
|
||||
GET DIAGNOSTICS deleted_count = ROW_COUNT;
|
||||
RETURN deleted_count;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
COMMENT ON FUNCTION cleanup_expired_academy_certificates IS 'Removes certificates that have expired beyond the specified number of days';
|
||||
125
admin-v2/deploy-and-ingest.sh
Executable file
125
admin-v2/deploy-and-ingest.sh
Executable file
@@ -0,0 +1,125 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# RAG DACH Vollabdeckung — Deploy & Ingest Script
|
||||
# Laeuft auf dem Mac Mini im Hintergrund (nohup)
|
||||
# ============================================================
|
||||
|
||||
set -e
|
||||
|
||||
LOG_FILE="/Users/benjaminadmin/Projekte/breakpilot-pwa/ingest-$(date +%Y%m%d-%H%M%S).log"
|
||||
PROJ="/Users/benjaminadmin/Projekte/breakpilot-pwa"
|
||||
DOCKER="/usr/local/bin/docker"
|
||||
COMPOSE="$DOCKER compose -f $PROJ/docker-compose.yml"
|
||||
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
|
||||
echo "============================================================"
|
||||
echo "RAG DACH Deploy & Ingest — Start: $(date)"
|
||||
echo "Logfile: $LOG_FILE"
|
||||
echo "============================================================"
|
||||
|
||||
# Phase 1: Check prerequisites
|
||||
echo ""
|
||||
echo "[1/6] Pruefe Docker-Services..."
|
||||
$COMPOSE ps qdrant embedding-service klausur-service 2>/dev/null || true
|
||||
|
||||
# Phase 2: Restart klausur-service to pick up new code
|
||||
echo ""
|
||||
echo "[2/6] Rebuilding klausur-service..."
|
||||
cd "$PROJ"
|
||||
$COMPOSE build --no-cache klausur-service
|
||||
echo "Build fertig."
|
||||
|
||||
echo ""
|
||||
echo "[3/6] Restarting klausur-service..."
|
||||
$COMPOSE up -d klausur-service
|
||||
echo "Warte 15 Sekunden auf Service-Start..."
|
||||
sleep 15
|
||||
|
||||
# Check if klausur-service is healthy
|
||||
echo "Pruefe klausur-service Health..."
|
||||
for i in 1 2 3 4 5; do
|
||||
if curl -sf http://127.0.0.1:8086/health > /dev/null 2>&1; then
|
||||
echo "klausur-service ist bereit."
|
||||
break
|
||||
fi
|
||||
echo "Warte auf klausur-service... ($i/5)"
|
||||
sleep 10
|
||||
done
|
||||
|
||||
# Phase 3: Run ingestion for new DACH laws only (not all — that would re-ingest existing ones)
|
||||
echo ""
|
||||
echo "[4/6] Starte Ingestion der neuen DACH-Gesetze (P1 zuerst)..."
|
||||
|
||||
# P1 — Deutschland
|
||||
echo ""
|
||||
echo "--- Deutschland P1 ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
DE_DDG DE_BGB_AGB DE_EGBGB DE_UWG DE_HGB_RET DE_AO_RET DE_TKG 2>&1 || echo "DE P1 hatte Fehler (non-fatal)"
|
||||
|
||||
# P1 — Oesterreich
|
||||
echo ""
|
||||
echo "--- Oesterreich P1 ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
AT_ECG AT_TKG AT_KSCHG AT_FAGG AT_UGB_RET AT_BAO_RET AT_MEDIENG 2>&1 || echo "AT P1 hatte Fehler (non-fatal)"
|
||||
|
||||
# P1 — Schweiz
|
||||
echo ""
|
||||
echo "--- Schweiz P1 ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
CH_DSV CH_OR_AGB CH_UWG CH_FMG 2>&1 || echo "CH P1 hatte Fehler (non-fatal)"
|
||||
|
||||
# 3 fehlgeschlagene Quellen nachholen
|
||||
echo ""
|
||||
echo "--- 3 fehlgeschlagene Quellen ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
LU_DPA_LAW DK_DATABESKYTTELSESLOVEN EDPB_GUIDELINES_1_2022 2>&1 || echo "Fix-3 hatte Fehler (non-fatal)"
|
||||
|
||||
echo ""
|
||||
echo "[5/6] Starte Ingestion P2 + P3..."
|
||||
|
||||
# P2 — Deutschland
|
||||
echo ""
|
||||
echo "--- Deutschland P2 ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
DE_PANGV DE_DLINFOV DE_BETRVG 2>&1 || echo "DE P2 hatte Fehler (non-fatal)"
|
||||
|
||||
# P2 — Oesterreich
|
||||
echo ""
|
||||
echo "--- Oesterreich P2 ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
AT_ABGB_AGB AT_UWG 2>&1 || echo "AT P2 hatte Fehler (non-fatal)"
|
||||
|
||||
# P2 — Schweiz
|
||||
echo ""
|
||||
echo "--- Schweiz P2 ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
CH_GEBUV CH_ZERTES 2>&1 || echo "CH P2 hatte Fehler (non-fatal)"
|
||||
|
||||
# P3
|
||||
echo ""
|
||||
echo "--- P3 (DE + CH) ---"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
DE_GESCHGEHG DE_BSIG DE_USTG_RET CH_ZGB_PERS 2>&1 || echo "P3 hatte Fehler (non-fatal)"
|
||||
|
||||
# Phase 4: Rebuild admin-v2 frontend
|
||||
echo ""
|
||||
echo "[6/6] Rebuilding admin-v2 Frontend..."
|
||||
$COMPOSE build --no-cache admin-v2
|
||||
$COMPOSE up -d admin-v2
|
||||
echo "admin-v2 neu gestartet."
|
||||
|
||||
# Phase 5: Status check
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo "FINAL STATUS CHECK"
|
||||
echo "============================================================"
|
||||
echo ""
|
||||
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --status 2>&1 || echo "Status-Check fehlgeschlagen"
|
||||
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo "Fertig: $(date)"
|
||||
echo "Logfile: $LOG_FILE"
|
||||
echo "============================================================"
|
||||
135
admin-v2/docker-compose.content.yml
Normal file
135
admin-v2/docker-compose.content.yml
Normal file
@@ -0,0 +1,135 @@
|
||||
# BreakPilot Content Service Stack
|
||||
# Usage: docker-compose -f docker-compose.yml -f docker-compose.content.yml up -d
|
||||
|
||||
services:
|
||||
# MinIO Object Storage (S3-compatible)
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: breakpilot-pwa-minio
|
||||
ports:
|
||||
- "9000:9000" # API
|
||||
- "9001:9001" # Console
|
||||
environment:
|
||||
MINIO_ROOT_USER: minioadmin
|
||||
MINIO_ROOT_PASSWORD: minioadmin123
|
||||
command: server /data --console-address ":9001"
|
||||
volumes:
|
||||
- minio_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
restart: unless-stopped
|
||||
|
||||
# Content Service Database (separate from main DB)
|
||||
content-db:
|
||||
image: postgres:16-alpine
|
||||
container_name: breakpilot-pwa-content-db
|
||||
ports:
|
||||
- "5433:5432"
|
||||
environment:
|
||||
POSTGRES_USER: breakpilot
|
||||
POSTGRES_PASSWORD: breakpilot123
|
||||
POSTGRES_DB: breakpilot_content
|
||||
volumes:
|
||||
- content_db_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U breakpilot -d breakpilot_content"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
restart: unless-stopped
|
||||
|
||||
# Content Service API
|
||||
content-service:
|
||||
build:
|
||||
context: ./backend/content_service
|
||||
dockerfile: Dockerfile
|
||||
container_name: breakpilot-pwa-content-service
|
||||
ports:
|
||||
- "8002:8002"
|
||||
environment:
|
||||
- CONTENT_DB_URL=postgresql://breakpilot:breakpilot123@content-db:5432/breakpilot_content
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- MINIO_ACCESS_KEY=minioadmin
|
||||
- MINIO_SECRET_KEY=minioadmin123
|
||||
- MINIO_SECURE=false
|
||||
- MINIO_BUCKET=breakpilot-content
|
||||
- CONSENT_SERVICE_URL=http://consent-service:8081
|
||||
- JWT_SECRET=${JWT_SECRET:-your-super-secret-jwt-key-change-in-production}
|
||||
- MATRIX_HOMESERVER=${MATRIX_HOMESERVER:-http://synapse:8008}
|
||||
- MATRIX_ACCESS_TOKEN=${MATRIX_ACCESS_TOKEN:-}
|
||||
depends_on:
|
||||
content-db:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
restart: unless-stopped
|
||||
|
||||
# H5P Interactive Content Service
|
||||
h5p-service:
|
||||
build:
|
||||
context: ./h5p-service
|
||||
dockerfile: Dockerfile
|
||||
container_name: breakpilot-pwa-h5p
|
||||
ports:
|
||||
- "8003:8080"
|
||||
environment:
|
||||
- H5P_STORAGE_PATH=/h5p-content
|
||||
- CONTENT_SERVICE_URL=http://content-service:8002
|
||||
volumes:
|
||||
- h5p_content:/h5p-content
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
restart: unless-stopped
|
||||
|
||||
# AI Content Generator Service
|
||||
ai-content-generator:
|
||||
build:
|
||||
context: ./ai-content-generator
|
||||
dockerfile: Dockerfile
|
||||
container_name: breakpilot-pwa-ai-generator
|
||||
ports:
|
||||
- "8004:8004"
|
||||
environment:
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
||||
- YOUTUBE_API_KEY=${YOUTUBE_API_KEY:-}
|
||||
- H5P_SERVICE_URL=http://h5p-service:8080
|
||||
- CONTENT_SERVICE_URL=http://content-service:8002
|
||||
- SERVICE_HOST=0.0.0.0
|
||||
- SERVICE_PORT=8004
|
||||
- MAX_UPLOAD_SIZE=10485760
|
||||
- MAX_CONCURRENT_JOBS=5
|
||||
- JOB_TIMEOUT=300
|
||||
volumes:
|
||||
- ai_generator_temp:/app/temp
|
||||
- ai_generator_uploads:/app/uploads
|
||||
depends_on:
|
||||
- h5p-service
|
||||
- content-service
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
minio_data:
|
||||
driver: local
|
||||
content_db_data:
|
||||
driver: local
|
||||
h5p_content:
|
||||
driver: local
|
||||
ai_generator_temp:
|
||||
driver: local
|
||||
ai_generator_uploads:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
breakpilot-pwa-network:
|
||||
external: true
|
||||
28
admin-v2/docker-compose.dev.yml
Normal file
28
admin-v2/docker-compose.dev.yml
Normal file
@@ -0,0 +1,28 @@
|
||||
# Development-specific overrides
|
||||
# Use with: docker-compose -f docker-compose.yml -f docker-compose.dev.yml up
|
||||
|
||||
services:
|
||||
backend:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
volumes:
|
||||
# Mount source code for hot-reload
|
||||
- ./backend:/app
|
||||
# Don't override the venv
|
||||
- /app/venv
|
||||
environment:
|
||||
- DEBUG=true
|
||||
command: uvicorn main:app --host 0.0.0.0 --port 8000 --reload
|
||||
|
||||
consent-service:
|
||||
# For development, you might want to use the local binary instead
|
||||
# Uncomment below to mount source and rebuild on changes
|
||||
# volumes:
|
||||
# - ./consent-service:/app
|
||||
environment:
|
||||
- GIN_MODE=debug
|
||||
|
||||
postgres:
|
||||
ports:
|
||||
- "5432:5432" # Expose for local tools
|
||||
108
admin-v2/docker-compose.override.yml
Normal file
108
admin-v2/docker-compose.override.yml
Normal file
@@ -0,0 +1,108 @@
|
||||
# ============================================
|
||||
# BreakPilot PWA - Development Overrides
|
||||
# ============================================
|
||||
# This file is AUTOMATICALLY loaded with: docker compose up
|
||||
# No need to specify -f flag for development!
|
||||
#
|
||||
# For staging: docker compose -f docker-compose.yml -f docker-compose.staging.yml up
|
||||
# ============================================
|
||||
|
||||
services:
|
||||
# ==========================================
|
||||
# Python Backend (FastAPI)
|
||||
# ==========================================
|
||||
backend:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
volumes:
|
||||
# Mount source code for hot-reload
|
||||
- ./backend:/app
|
||||
# Don't override the venv
|
||||
- /app/venv
|
||||
environment:
|
||||
- DEBUG=true
|
||||
- ENVIRONMENT=development
|
||||
- LOG_LEVEL=debug
|
||||
command: uvicorn main:app --host 0.0.0.0 --port 8000 --reload
|
||||
|
||||
# ==========================================
|
||||
# Go Consent Service
|
||||
# ==========================================
|
||||
consent-service:
|
||||
environment:
|
||||
- GIN_MODE=debug
|
||||
- ENVIRONMENT=development
|
||||
- LOG_LEVEL=debug
|
||||
|
||||
# ==========================================
|
||||
# Go School Service
|
||||
# ==========================================
|
||||
school-service:
|
||||
environment:
|
||||
- GIN_MODE=debug
|
||||
- ENVIRONMENT=development
|
||||
|
||||
# ==========================================
|
||||
# Go Billing Service
|
||||
# ==========================================
|
||||
billing-service:
|
||||
environment:
|
||||
- GIN_MODE=debug
|
||||
- ENVIRONMENT=development
|
||||
|
||||
# ==========================================
|
||||
# Klausur Service (Python + React)
|
||||
# ==========================================
|
||||
klausur-service:
|
||||
environment:
|
||||
- DEBUG=true
|
||||
- ENVIRONMENT=development
|
||||
|
||||
# ==========================================
|
||||
# Website (Next.js)
|
||||
# ==========================================
|
||||
website:
|
||||
environment:
|
||||
- NODE_ENV=development
|
||||
|
||||
# ==========================================
|
||||
# PostgreSQL
|
||||
# ==========================================
|
||||
postgres:
|
||||
ports:
|
||||
- "5432:5432" # Expose for local DB tools
|
||||
environment:
|
||||
- POSTGRES_DB=${POSTGRES_DB:-breakpilot_dev}
|
||||
|
||||
# ==========================================
|
||||
# MinIO (Object Storage)
|
||||
# ==========================================
|
||||
minio:
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001" # Console
|
||||
|
||||
# ==========================================
|
||||
# Qdrant (Vector DB)
|
||||
# ==========================================
|
||||
qdrant:
|
||||
ports:
|
||||
- "6333:6333"
|
||||
- "6334:6334"
|
||||
|
||||
# ==========================================
|
||||
# Mailpit (Email Testing)
|
||||
# ==========================================
|
||||
mailpit:
|
||||
ports:
|
||||
- "8025:8025" # Web UI
|
||||
- "1025:1025" # SMTP
|
||||
|
||||
# ==========================================
|
||||
# DSMS Gateway
|
||||
# ==========================================
|
||||
dsms-gateway:
|
||||
environment:
|
||||
- DEBUG=true
|
||||
- ENVIRONMENT=development
|
||||
133
admin-v2/docker-compose.staging.yml
Normal file
133
admin-v2/docker-compose.staging.yml
Normal file
@@ -0,0 +1,133 @@
|
||||
# ============================================
|
||||
# BreakPilot PWA - Staging Overrides
|
||||
# ============================================
|
||||
# Usage: docker compose -f docker-compose.yml -f docker-compose.staging.yml up -d
|
||||
#
|
||||
# Or use the helper script:
|
||||
# ./scripts/start.sh staging
|
||||
# ============================================
|
||||
|
||||
services:
|
||||
# ==========================================
|
||||
# Python Backend (FastAPI)
|
||||
# ==========================================
|
||||
backend:
|
||||
environment:
|
||||
- DEBUG=false
|
||||
- ENVIRONMENT=staging
|
||||
- LOG_LEVEL=info
|
||||
restart: unless-stopped
|
||||
# No hot-reload in staging
|
||||
command: uvicorn main:app --host 0.0.0.0 --port 8000
|
||||
|
||||
# ==========================================
|
||||
# Go Consent Service
|
||||
# ==========================================
|
||||
consent-service:
|
||||
environment:
|
||||
- GIN_MODE=release
|
||||
- ENVIRONMENT=staging
|
||||
- LOG_LEVEL=info
|
||||
restart: unless-stopped
|
||||
|
||||
# ==========================================
|
||||
# Go School Service
|
||||
# ==========================================
|
||||
school-service:
|
||||
environment:
|
||||
- GIN_MODE=release
|
||||
- ENVIRONMENT=staging
|
||||
restart: unless-stopped
|
||||
|
||||
# ==========================================
|
||||
# Go Billing Service
|
||||
# ==========================================
|
||||
billing-service:
|
||||
environment:
|
||||
- GIN_MODE=release
|
||||
- ENVIRONMENT=staging
|
||||
restart: unless-stopped
|
||||
|
||||
# ==========================================
|
||||
# Klausur Service (Python + React)
|
||||
# ==========================================
|
||||
klausur-service:
|
||||
environment:
|
||||
- DEBUG=false
|
||||
- ENVIRONMENT=staging
|
||||
restart: unless-stopped
|
||||
|
||||
# ==========================================
|
||||
# Website (Next.js)
|
||||
# ==========================================
|
||||
website:
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
restart: unless-stopped
|
||||
|
||||
# ==========================================
|
||||
# PostgreSQL (Separate Database for Staging)
|
||||
# ==========================================
|
||||
postgres:
|
||||
ports:
|
||||
- "5433:5432" # Different port for staging!
|
||||
environment:
|
||||
- POSTGRES_DB=${POSTGRES_DB:-breakpilot_staging}
|
||||
volumes:
|
||||
- breakpilot_staging_postgres:/var/lib/postgresql/data
|
||||
|
||||
# ==========================================
|
||||
# MinIO (Object Storage - Different Ports)
|
||||
# ==========================================
|
||||
minio:
|
||||
ports:
|
||||
- "9002:9000"
|
||||
- "9003:9001"
|
||||
volumes:
|
||||
- breakpilot_staging_minio:/data
|
||||
|
||||
# ==========================================
|
||||
# Qdrant (Vector DB - Different Ports)
|
||||
# ==========================================
|
||||
qdrant:
|
||||
ports:
|
||||
- "6335:6333"
|
||||
- "6336:6334"
|
||||
volumes:
|
||||
- breakpilot_staging_qdrant:/qdrant/storage
|
||||
|
||||
# ==========================================
|
||||
# Mailpit (Still using Mailpit for Safety)
|
||||
# ==========================================
|
||||
mailpit:
|
||||
ports:
|
||||
- "8026:8025" # Different Web UI port
|
||||
- "1026:1025" # Different SMTP port
|
||||
|
||||
# ==========================================
|
||||
# DSMS Gateway
|
||||
# ==========================================
|
||||
dsms-gateway:
|
||||
environment:
|
||||
- DEBUG=false
|
||||
- ENVIRONMENT=staging
|
||||
restart: unless-stopped
|
||||
|
||||
# ==========================================
|
||||
# Enable Backup Service in Staging
|
||||
# ==========================================
|
||||
backup:
|
||||
profiles: [] # Remove profile restriction = always start
|
||||
environment:
|
||||
- PGDATABASE=breakpilot_staging
|
||||
|
||||
# ==========================================
|
||||
# Separate Volumes for Staging
|
||||
# ==========================================
|
||||
volumes:
|
||||
breakpilot_staging_postgres:
|
||||
name: breakpilot_staging_postgres
|
||||
breakpilot_staging_minio:
|
||||
name: breakpilot_staging_minio
|
||||
breakpilot_staging_qdrant:
|
||||
name: breakpilot_staging_qdrant
|
||||
153
admin-v2/docker-compose.test.yml
Normal file
153
admin-v2/docker-compose.test.yml
Normal file
@@ -0,0 +1,153 @@
|
||||
# BreakPilot PWA - Test-Infrastruktur
|
||||
#
|
||||
# Vollstaendige Integration-Test Umgebung fuer CI/CD Pipeline.
|
||||
# Startet alle Services isoliert fuer Integration-Tests.
|
||||
#
|
||||
# Verwendung:
|
||||
# docker compose -f docker-compose.test.yml up -d
|
||||
# docker compose -f docker-compose.test.yml down -v
|
||||
#
|
||||
# Verbindungen:
|
||||
# PostgreSQL: localhost:55432 (breakpilot_test/breakpilot/breakpilot)
|
||||
# Valkey/Redis: localhost:56379
|
||||
# Consent Service: localhost:58081
|
||||
# Backend: localhost:58000
|
||||
# Mailpit Web: localhost:58025
|
||||
# Mailpit SMTP: localhost:51025
|
||||
|
||||
version: "3.9"
|
||||
|
||||
services:
|
||||
# ========================================
|
||||
# Datenbank-Services
|
||||
# ========================================
|
||||
|
||||
postgres-test:
|
||||
image: postgres:16-alpine
|
||||
container_name: breakpilot-postgres-test
|
||||
environment:
|
||||
POSTGRES_DB: breakpilot_test
|
||||
POSTGRES_USER: breakpilot
|
||||
POSTGRES_PASSWORD: breakpilot_test
|
||||
ports:
|
||||
- "55432:5432"
|
||||
volumes:
|
||||
- postgres_test_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U breakpilot -d breakpilot_test"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- breakpilot-test-network
|
||||
restart: unless-stopped
|
||||
|
||||
valkey-test:
|
||||
image: valkey/valkey:7-alpine
|
||||
container_name: breakpilot-valkey-test
|
||||
ports:
|
||||
- "56379:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- breakpilot-test-network
|
||||
restart: unless-stopped
|
||||
|
||||
# ========================================
|
||||
# Application Services
|
||||
# ========================================
|
||||
|
||||
# Consent Service (Go)
|
||||
consent-service-test:
|
||||
build:
|
||||
context: ./consent-service
|
||||
dockerfile: Dockerfile
|
||||
container_name: breakpilot-consent-service-test
|
||||
ports:
|
||||
- "58081:8081"
|
||||
depends_on:
|
||||
postgres-test:
|
||||
condition: service_healthy
|
||||
valkey-test:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8081/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
environment:
|
||||
- DATABASE_URL=postgres://breakpilot:breakpilot_test@postgres-test:5432/breakpilot_test
|
||||
- VALKEY_URL=redis://valkey-test:6379
|
||||
- REDIS_URL=redis://valkey-test:6379
|
||||
- JWT_SECRET=test-jwt-secret-for-integration-tests
|
||||
- ENVIRONMENT=test
|
||||
- LOG_LEVEL=debug
|
||||
networks:
|
||||
- breakpilot-test-network
|
||||
restart: unless-stopped
|
||||
|
||||
# Backend (Python FastAPI)
|
||||
backend-test:
|
||||
build:
|
||||
context: ./backend
|
||||
dockerfile: Dockerfile
|
||||
container_name: breakpilot-backend-test
|
||||
ports:
|
||||
- "58000:8000"
|
||||
depends_on:
|
||||
postgres-test:
|
||||
condition: service_healthy
|
||||
valkey-test:
|
||||
condition: service_healthy
|
||||
consent-service-test:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 45s
|
||||
environment:
|
||||
- DATABASE_URL=postgresql://breakpilot:breakpilot_test@postgres-test:5432/breakpilot_test
|
||||
- CONSENT_SERVICE_URL=http://consent-service-test:8081
|
||||
- VALKEY_URL=redis://valkey-test:6379
|
||||
- REDIS_URL=redis://valkey-test:6379
|
||||
- JWT_SECRET=test-jwt-secret-for-integration-tests
|
||||
- ENVIRONMENT=test
|
||||
- SMTP_HOST=mailpit-test
|
||||
- SMTP_PORT=1025
|
||||
- SKIP_INTEGRATION_TESTS=false
|
||||
networks:
|
||||
- breakpilot-test-network
|
||||
restart: unless-stopped
|
||||
|
||||
# ========================================
|
||||
# Development/Testing Tools
|
||||
# ========================================
|
||||
|
||||
# Mailpit (E-Mail Testing)
|
||||
mailpit-test:
|
||||
image: axllent/mailpit:latest
|
||||
container_name: breakpilot-mailpit-test
|
||||
ports:
|
||||
- "58025:8025" # Web UI
|
||||
- "51025:1025" # SMTP
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:8025/api/v1/info"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- breakpilot-test-network
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
breakpilot-test-network:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
postgres_test_data:
|
||||
98
admin-v2/docker-compose.vault.yml
Normal file
98
admin-v2/docker-compose.vault.yml
Normal file
@@ -0,0 +1,98 @@
|
||||
# HashiCorp Vault Configuration for BreakPilot
|
||||
#
|
||||
# Usage:
|
||||
# Development mode (unsealed, no auth required):
|
||||
# docker-compose -f docker-compose.vault.yml up -d vault
|
||||
#
|
||||
# Production mode:
|
||||
# docker-compose -f docker-compose.vault.yml --profile production up -d
|
||||
#
|
||||
# After starting Vault in dev mode:
|
||||
# export VAULT_ADDR=http://localhost:8200
|
||||
# export VAULT_TOKEN=breakpilot-dev-token
|
||||
#
|
||||
# License: HashiCorp Vault is BSL 1.1 (open source for non-commercial use)
|
||||
# Vault clients (hvac) are Apache-2.0
|
||||
|
||||
services:
|
||||
# HashiCorp Vault - Secrets Management
|
||||
vault:
|
||||
image: hashicorp/vault:1.15
|
||||
container_name: breakpilot-pwa-vault
|
||||
ports:
|
||||
- "8200:8200"
|
||||
environment:
|
||||
# Development mode settings
|
||||
VAULT_DEV_ROOT_TOKEN_ID: ${VAULT_DEV_TOKEN:-breakpilot-dev-token}
|
||||
VAULT_DEV_LISTEN_ADDRESS: "0.0.0.0:8200"
|
||||
VAULT_ADDR: "http://127.0.0.1:8200"
|
||||
VAULT_API_ADDR: "http://0.0.0.0:8200"
|
||||
cap_add:
|
||||
- IPC_LOCK # Required for mlock
|
||||
volumes:
|
||||
- vault_data:/vault/data
|
||||
- vault_logs:/vault/logs
|
||||
- ./vault/config:/vault/config:ro
|
||||
- ./vault/policies:/vault/policies:ro
|
||||
command: server -dev -dev-root-token-id=${VAULT_DEV_TOKEN:-breakpilot-dev-token}
|
||||
healthcheck:
|
||||
test: ["CMD", "vault", "status"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
restart: unless-stopped
|
||||
|
||||
# Vault Agent for automatic secret injection (production)
|
||||
vault-agent:
|
||||
image: hashicorp/vault:1.15
|
||||
container_name: breakpilot-pwa-vault-agent
|
||||
profiles:
|
||||
- production
|
||||
depends_on:
|
||||
vault:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
VAULT_ADDR: "http://vault:8200"
|
||||
volumes:
|
||||
- ./vault/agent-config.hcl:/vault/config/agent-config.hcl:ro
|
||||
- vault_agent_secrets:/vault/secrets
|
||||
command: agent -config=/vault/config/agent-config.hcl
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
restart: unless-stopped
|
||||
|
||||
# Vault initializer - Seeds secrets in development
|
||||
vault-init:
|
||||
image: hashicorp/vault:1.15
|
||||
container_name: breakpilot-pwa-vault-init
|
||||
depends_on:
|
||||
vault:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
VAULT_ADDR: "http://vault:8200"
|
||||
VAULT_TOKEN: ${VAULT_DEV_TOKEN:-breakpilot-dev-token}
|
||||
volumes:
|
||||
- ./vault/init-secrets.sh:/vault/init-secrets.sh:ro
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
sleep 5
|
||||
chmod +x /vault/init-secrets.sh
|
||||
/vault/init-secrets.sh
|
||||
echo "Vault initialized with development secrets"
|
||||
networks:
|
||||
- breakpilot-pwa-network
|
||||
|
||||
volumes:
|
||||
vault_data:
|
||||
name: breakpilot_vault_data
|
||||
vault_logs:
|
||||
name: breakpilot_vault_logs
|
||||
vault_agent_secrets:
|
||||
name: breakpilot_vault_agent_secrets
|
||||
|
||||
networks:
|
||||
breakpilot-pwa-network:
|
||||
external: true
|
||||
1832
admin-v2/docker-compose.yml
Normal file
1832
admin-v2/docker-compose.yml
Normal file
File diff suppressed because it is too large
Load Diff
101
admin-v2/mkdocs.yml
Normal file
101
admin-v2/mkdocs.yml
Normal file
@@ -0,0 +1,101 @@
|
||||
site_name: Breakpilot Dokumentation
|
||||
site_url: https://macmini:8008
|
||||
docs_dir: docs-src
|
||||
site_dir: docs-site
|
||||
|
||||
theme:
|
||||
name: material
|
||||
language: de
|
||||
palette:
|
||||
- scheme: default
|
||||
primary: teal
|
||||
toggle:
|
||||
icon: material/brightness-7
|
||||
name: Dark Mode aktivieren
|
||||
- scheme: slate
|
||||
primary: teal
|
||||
toggle:
|
||||
icon: material/brightness-4
|
||||
name: Light Mode aktivieren
|
||||
features:
|
||||
- search.highlight
|
||||
- search.suggest
|
||||
- navigation.tabs
|
||||
- navigation.sections
|
||||
- navigation.expand
|
||||
- navigation.top
|
||||
- content.code.copy
|
||||
- content.tabs.link
|
||||
- toc.follow
|
||||
|
||||
plugins:
|
||||
- search:
|
||||
lang: de
|
||||
|
||||
markdown_extensions:
|
||||
- admonition
|
||||
- pymdownx.details
|
||||
- pymdownx.superfences:
|
||||
custom_fences:
|
||||
- name: mermaid
|
||||
class: mermaid
|
||||
format: !!python/name:pymdownx.superfences.fence_code_format
|
||||
- pymdownx.tabbed:
|
||||
alternate_style: true
|
||||
- pymdownx.highlight:
|
||||
anchor_linenums: true
|
||||
- pymdownx.inlinehilite
|
||||
- pymdownx.snippets
|
||||
- tables
|
||||
- attr_list
|
||||
- md_in_html
|
||||
- toc:
|
||||
permalink: true
|
||||
|
||||
extra:
|
||||
social:
|
||||
- icon: fontawesome/brands/github
|
||||
link: http://macmini:3003/breakpilot/breakpilot-pwa
|
||||
|
||||
nav:
|
||||
- Start: index.md
|
||||
- Erste Schritte:
|
||||
- Umgebung einrichten: getting-started/environment-setup.md
|
||||
- Mac Mini Setup: getting-started/mac-mini-setup.md
|
||||
- Architektur:
|
||||
- Systemuebersicht: architecture/system-architecture.md
|
||||
- Auth-System: architecture/auth-system.md
|
||||
- Mail-RBAC: architecture/mail-rbac-architecture.md
|
||||
- Multi-Agent: architecture/multi-agent.md
|
||||
- Secrets Management: architecture/secrets-management.md
|
||||
- DevSecOps: architecture/devsecops.md
|
||||
- Environments: architecture/environments.md
|
||||
- Zeugnis-System: architecture/zeugnis-system.md
|
||||
- Services:
|
||||
- KI-Daten-Pipeline:
|
||||
- Uebersicht: services/ki-daten-pipeline/index.md
|
||||
- Architektur: services/ki-daten-pipeline/architecture.md
|
||||
- Klausur-Service:
|
||||
- Uebersicht: services/klausur-service/index.md
|
||||
- BYOEH Systemerklaerung: services/klausur-service/byoeh-system-erklaerung.md
|
||||
- BYOEH Architektur: services/klausur-service/BYOEH-Architecture.md
|
||||
- BYOEH Developer Guide: services/klausur-service/BYOEH-Developer-Guide.md
|
||||
- NiBiS Pipeline: services/klausur-service/NiBiS-Ingestion-Pipeline.md
|
||||
- OCR Labeling: services/klausur-service/OCR-Labeling-Spec.md
|
||||
- OCR Compare: services/klausur-service/OCR-Compare.md
|
||||
- RAG Admin: services/klausur-service/RAG-Admin-Spec.md
|
||||
- Worksheet Editor: services/klausur-service/Worksheet-Editor-Architecture.md
|
||||
- Voice-Service: services/voice-service/index.md
|
||||
- Agent-Core: services/agent-core/index.md
|
||||
- AI-Compliance-SDK:
|
||||
- Uebersicht: services/ai-compliance-sdk/index.md
|
||||
- Architektur: services/ai-compliance-sdk/ARCHITECTURE.md
|
||||
- Developer Guide: services/ai-compliance-sdk/DEVELOPER.md
|
||||
- Auditor Dokumentation: services/ai-compliance-sdk/AUDITOR_DOCUMENTATION.md
|
||||
- SBOM: services/ai-compliance-sdk/SBOM.md
|
||||
- API:
|
||||
- Backend API: api/backend-api.md
|
||||
- Entwicklung:
|
||||
- Testing: development/testing.md
|
||||
- Dokumentation: development/documentation.md
|
||||
- CI/CD Pipeline: development/ci-cd-pipeline.md
|
||||
5694
admin-v2/package-lock.json
generated
5694
admin-v2/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
66
admin-v2/run-ingestion.sh
Executable file
66
admin-v2/run-ingestion.sh
Executable file
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# RAG DACH Ingestion — Nur Ingestion (Builds schon fertig)
|
||||
# ============================================================
|
||||
|
||||
PROJ="/Users/benjaminadmin/Projekte/breakpilot-pwa"
|
||||
DOCKER="/usr/local/bin/docker"
|
||||
COMPOSE="$DOCKER compose -f $PROJ/docker-compose.yml"
|
||||
LOG_FILE="$PROJ/ingest-$(date +%Y%m%d-%H%M%S).log"
|
||||
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
|
||||
echo "============================================================"
|
||||
echo "RAG DACH Ingestion — Start: $(date)"
|
||||
echo "Logfile: $LOG_FILE"
|
||||
echo "============================================================"
|
||||
|
||||
# Health Check (via docker exec, Port nicht auf Host exponiert)
|
||||
echo ""
|
||||
echo "[1/5] Pruefe klausur-service..."
|
||||
if ! $COMPOSE exec -T klausur-service python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8086/health')" 2>/dev/null; then
|
||||
echo "FEHLER: klausur-service nicht erreichbar!"
|
||||
exit 1
|
||||
fi
|
||||
echo "klausur-service ist bereit."
|
||||
|
||||
# P1 — Deutschland
|
||||
echo ""
|
||||
echo "[2/5] Ingestion P1 — Deutschland (7 Gesetze)..."
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
DE_DDG DE_BGB_AGB DE_EGBGB DE_UWG DE_HGB_RET DE_AO_RET DE_TKG 2>&1 || echo "DE P1 hatte Fehler"
|
||||
|
||||
# P1 — Oesterreich
|
||||
echo ""
|
||||
echo "[3/5] Ingestion P1 — Oesterreich (7 Gesetze)..."
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
AT_ECG AT_TKG AT_KSCHG AT_FAGG AT_UGB_RET AT_BAO_RET AT_MEDIENG 2>&1 || echo "AT P1 hatte Fehler"
|
||||
|
||||
# P1 — Schweiz
|
||||
echo ""
|
||||
echo "[4/5] Ingestion P1 — Schweiz (4 Gesetze)..."
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
CH_DSV CH_OR_AGB CH_UWG CH_FMG 2>&1 || echo "CH P1 hatte Fehler"
|
||||
|
||||
# 3 fehlgeschlagene Quellen + P2 + P3
|
||||
echo ""
|
||||
echo "[5/5] Ingestion P2/P3 + Fixes (14 Gesetze)..."
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --ingest \
|
||||
LU_DPA_LAW DK_DATABESKYTTELSESLOVEN EDPB_GUIDELINES_1_2022 \
|
||||
DE_PANGV DE_DLINFOV DE_BETRVG \
|
||||
AT_ABGB_AGB AT_UWG \
|
||||
CH_GEBUV CH_ZERTES \
|
||||
DE_GESCHGEHG DE_BSIG DE_USTG_RET CH_ZGB_PERS 2>&1 || echo "P2/P3 hatte Fehler"
|
||||
|
||||
# Status
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo "FINAL STATUS CHECK"
|
||||
echo "============================================================"
|
||||
$COMPOSE exec -T klausur-service python -m legal_corpus_ingestion --status 2>&1
|
||||
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo "Fertig: $(date)"
|
||||
echo "Logfile: $LOG_FILE"
|
||||
echo "============================================================"
|
||||
Reference in New Issue
Block a user