From cb57a341292d33ad8f3d4157bb8df486a53bdb52 Mon Sep 17 00:00:00 2001 From: Benjamin Boenisch Date: Sun, 15 Feb 2026 10:56:02 +0100 Subject: [PATCH] Add Woodpecker CI/CD pipeline - Lint: golangci-lint (ai-compliance-sdk), ruff (Python), next lint (Node.js) - Tests: Go tests, pytest for backend-compliance, document-crawler, dsms-gateway - Build: Docker images for all services - Security: SBOM generation + vulnerability scanning - Deploy: manual docker compose deployment --- .woodpecker/main.yml | 418 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 418 insertions(+) create mode 100644 .woodpecker/main.yml diff --git a/.woodpecker/main.yml b/.woodpecker/main.yml new file mode 100644 index 0000000..64ab31a --- /dev/null +++ b/.woodpecker/main.yml @@ -0,0 +1,418 @@ +# Woodpecker CI Main Pipeline +# BreakPilot Compliance - CI/CD Pipeline +# +# Plattform: ARM64 (Apple Silicon Mac Mini) +# +# Services: +# Go: ai-compliance-sdk +# Python: backend-compliance, document-crawler, dsms-gateway +# Node.js: admin-compliance, developer-portal +# +# Strategie: +# - Lint bei PRs +# - Tests laufen bei JEDEM Push/PR +# - Test-Ergebnisse werden an Dashboard gesendet +# - Builds/Scans laufen nur bei Tags oder manuell +# - Deployment nur manuell (Sicherheit) + +when: + - event: [push, pull_request, manual, tag] + branch: [main, develop] + +clone: + git: + image: woodpeckerci/plugin-git + settings: + depth: 1 + extra_hosts: + - macmini:192.168.178.100 + +variables: + - &golang_image golang:1.23-alpine + - &python_image python:3.12-slim + - &nodejs_image node:20-alpine + - &docker_image docker:27-cli + +steps: + # ======================================== + # STAGE 1: Lint (nur bei PRs) + # ======================================== + + go-lint: + image: golangci/golangci-lint:v1.55-alpine + commands: + - | + if [ -d "ai-compliance-sdk" ]; then + cd ai-compliance-sdk && golangci-lint run --timeout 5m ./... + fi + when: + event: pull_request + + python-lint: + image: *python_image + commands: + - pip install --quiet ruff + - | + for svc in backend-compliance document-crawler dsms-gateway; do + if [ -d "$svc" ]; then + echo "=== Linting $svc ===" + ruff check "$svc/" --output-format=github || true + fi + done + when: + event: pull_request + + nodejs-lint: + image: *nodejs_image + commands: + - | + for svc in admin-compliance developer-portal; do + if [ -d "$svc" ]; then + echo "=== Linting $svc ===" + cd "$svc" + npm ci --silent 2>/dev/null || npm install --silent + npx next lint || true + cd .. + fi + done + when: + event: pull_request + + # ======================================== + # STAGE 2: Unit Tests mit JSON-Ausgabe + # Ergebnisse werden im Workspace gespeichert (.ci-results/) + # ======================================== + + test-go-ai-compliance: + image: *golang_image + environment: + CGO_ENABLED: "0" + commands: + - | + set -euo pipefail + apk add --no-cache jq bash + mkdir -p .ci-results + + if [ ! -d "ai-compliance-sdk" ]; then + echo '{"service":"ai-compliance-sdk","framework":"go","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-ai-compliance.json + echo "WARNUNG: ai-compliance-sdk Verzeichnis nicht gefunden" + exit 0 + fi + + cd ai-compliance-sdk + set +e + go test -v -json -coverprofile=coverage.out ./... 2>&1 | tee ../.ci-results/test-ai-compliance.json + TEST_EXIT=$? + set -e + + JSON_FILE="../.ci-results/test-ai-compliance.json" + if grep -q '^{' "$JSON_FILE" 2>/dev/null; then + TOTAL=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="run" and .Test != null)] | length') + PASSED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="pass" and .Test != null)] | length') + FAILED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="fail" and .Test != null)] | length') + SKIPPED=$(grep '^{' "$JSON_FILE" | jq -s '[.[] | select(.Action=="skip" and .Test != null)] | length') + else + echo "WARNUNG: Keine JSON-Zeilen in $JSON_FILE gefunden (Build-Fehler?)" + TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0 + fi + + COVERAGE=$(go tool cover -func=coverage.out 2>/dev/null | tail -1 | awk '{print $3}' | tr -d '%' || echo "0") + [ -z "$COVERAGE" ] && COVERAGE=0 + + echo "{\"service\":\"ai-compliance-sdk\",\"framework\":\"go\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":$COVERAGE}" > ../.ci-results/results-ai-compliance.json + cat ../.ci-results/results-ai-compliance.json + + # Backlog-Strategie: Fehler werden gemeldet aber Pipeline laeuft weiter + if [ "$FAILED" -gt "0" ]; then + echo "WARNUNG: $FAILED Tests fehlgeschlagen - werden ins Backlog geschrieben" + fi + + test-python-backend-compliance: + image: *python_image + environment: + CI: "true" + commands: + - | + set -uo pipefail + mkdir -p .ci-results + + if [ ! -d "backend-compliance" ]; then + echo '{"service":"backend-compliance","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-backend-compliance.json + echo "WARNUNG: backend-compliance Verzeichnis nicht gefunden" + exit 0 + fi + + cd backend-compliance + export PYTHONPATH="$(pwd):${PYTHONPATH:-}" + pip install --quiet --no-cache-dir -r requirements.txt 2>/dev/null || pip install --quiet --no-cache-dir fastapi uvicorn pytest pytest-json-report + pip install --quiet --no-cache-dir pytest-json-report + + set +e + python -m pytest tests/ -v --tb=short --json-report --json-report-file=../.ci-results/test-backend-compliance.json + TEST_EXIT=$? + set -e + + if [ -f ../.ci-results/test-backend-compliance.json ]; then + TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend-compliance.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0") + PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend-compliance.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0") + FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend-compliance.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0") + SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-backend-compliance.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0") + else + TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0 + fi + + echo "{\"service\":\"backend-compliance\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-backend-compliance.json + cat ../.ci-results/results-backend-compliance.json + + if [ "$TEST_EXIT" -ne "0" ]; then exit 1; fi + + test-python-document-crawler: + image: *python_image + environment: + CI: "true" + commands: + - | + set -uo pipefail + mkdir -p .ci-results + + if [ ! -d "document-crawler" ]; then + echo '{"service":"document-crawler","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-document-crawler.json + echo "WARNUNG: document-crawler Verzeichnis nicht gefunden" + exit 0 + fi + + cd document-crawler + export PYTHONPATH="$(pwd):${PYTHONPATH:-}" + pip install --quiet --no-cache-dir -r requirements.txt 2>/dev/null || pip install --quiet --no-cache-dir fastapi uvicorn pytest pytest-json-report + pip install --quiet --no-cache-dir pytest-json-report + + set +e + python -m pytest tests/ -v --tb=short --json-report --json-report-file=../.ci-results/test-document-crawler.json + TEST_EXIT=$? + set -e + + if [ -f ../.ci-results/test-document-crawler.json ]; then + TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-document-crawler.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0") + PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-document-crawler.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0") + FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-document-crawler.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0") + SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-document-crawler.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0") + else + TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0 + fi + + echo "{\"service\":\"document-crawler\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-document-crawler.json + cat ../.ci-results/results-document-crawler.json + + if [ "$TEST_EXIT" -ne "0" ]; then exit 1; fi + + test-python-dsms-gateway: + image: *python_image + environment: + CI: "true" + commands: + - | + set -uo pipefail + mkdir -p .ci-results + + if [ ! -d "dsms-gateway" ]; then + echo '{"service":"dsms-gateway","framework":"pytest","total":0,"passed":0,"failed":0,"skipped":0,"coverage":0}' > .ci-results/results-dsms-gateway.json + echo "WARNUNG: dsms-gateway Verzeichnis nicht gefunden" + exit 0 + fi + + cd dsms-gateway + export PYTHONPATH="$(pwd):${PYTHONPATH:-}" + pip install --quiet --no-cache-dir -r requirements.txt 2>/dev/null || pip install --quiet --no-cache-dir fastapi uvicorn pytest pytest-json-report + pip install --quiet --no-cache-dir pytest-json-report + + set +e + python -m pytest tests/ -v --tb=short --json-report --json-report-file=../.ci-results/test-dsms-gateway.json + TEST_EXIT=$? + set -e + + if [ -f ../.ci-results/test-dsms-gateway.json ]; then + TOTAL=$(python3 -c "import json; d=json.load(open('../.ci-results/test-dsms-gateway.json')); print(d.get('summary',{}).get('total',0))" 2>/dev/null || echo "0") + PASSED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-dsms-gateway.json')); print(d.get('summary',{}).get('passed',0))" 2>/dev/null || echo "0") + FAILED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-dsms-gateway.json')); print(d.get('summary',{}).get('failed',0))" 2>/dev/null || echo "0") + SKIPPED=$(python3 -c "import json; d=json.load(open('../.ci-results/test-dsms-gateway.json')); print(d.get('summary',{}).get('skipped',0))" 2>/dev/null || echo "0") + else + TOTAL=0; PASSED=0; FAILED=0; SKIPPED=0 + fi + + echo "{\"service\":\"dsms-gateway\",\"framework\":\"pytest\",\"total\":$TOTAL,\"passed\":$PASSED,\"failed\":$FAILED,\"skipped\":$SKIPPED,\"coverage\":0}" > ../.ci-results/results-dsms-gateway.json + cat ../.ci-results/results-dsms-gateway.json + + if [ "$TEST_EXIT" -ne "0" ]; then exit 1; fi + + # ======================================== + # STAGE 3: Test-Ergebnisse an Dashboard senden + # ======================================== + + report-test-results: + image: curlimages/curl:8.10.1 + commands: + - | + set -uo pipefail + echo "=== Sende Test-Ergebnisse an Dashboard ===" + echo "Pipeline Status: ${CI_PIPELINE_STATUS:-unknown}" + ls -la .ci-results/ || echo "Verzeichnis nicht gefunden" + + PIPELINE_STATUS="${CI_PIPELINE_STATUS:-unknown}" + + for f in .ci-results/results-*.json; do + [ -f "$f" ] || continue + echo "Sending: $f" + curl -f -sS -X POST "http://backend:8000/api/tests/ci-result" \ + -H "Content-Type: application/json" \ + -d "{ + \"pipeline_id\": \"${CI_PIPELINE_NUMBER}\", + \"commit\": \"${CI_COMMIT_SHA}\", + \"branch\": \"${CI_COMMIT_BRANCH}\", + \"repo\": \"breakpilot-compliance\", + \"status\": \"${PIPELINE_STATUS}\", + \"test_results\": $(cat "$f") + }" || echo "WARNUNG: Konnte $f nicht senden" + done + + echo "=== Test-Ergebnisse gesendet ===" + when: + status: [success, failure] + depends_on: + - test-go-ai-compliance + - test-python-backend-compliance + - test-python-document-crawler + - test-python-dsms-gateway + + # ======================================== + # STAGE 4: Build & Security (nur Tags/manuell) + # ======================================== + + build-ai-compliance-sdk: + image: *docker_image + commands: + - | + if [ -d ./ai-compliance-sdk ]; then + docker build -t breakpilot/ai-compliance-sdk:${CI_COMMIT_SHA:0:8} ./ai-compliance-sdk + docker tag breakpilot/ai-compliance-sdk:${CI_COMMIT_SHA:0:8} breakpilot/ai-compliance-sdk:latest + echo "Built breakpilot/ai-compliance-sdk:${CI_COMMIT_SHA:0:8}" + else + echo "ai-compliance-sdk Verzeichnis nicht gefunden - ueberspringe" + fi + when: + - event: tag + - event: manual + + build-backend-compliance: + image: *docker_image + commands: + - | + if [ -d ./backend-compliance ]; then + docker build -t breakpilot/backend-compliance:${CI_COMMIT_SHA:0:8} ./backend-compliance + docker tag breakpilot/backend-compliance:${CI_COMMIT_SHA:0:8} breakpilot/backend-compliance:latest + echo "Built breakpilot/backend-compliance:${CI_COMMIT_SHA:0:8}" + else + echo "backend-compliance Verzeichnis nicht gefunden - ueberspringe" + fi + when: + - event: tag + - event: manual + + build-document-crawler: + image: *docker_image + commands: + - | + if [ -d ./document-crawler ]; then + docker build -t breakpilot/document-crawler:${CI_COMMIT_SHA:0:8} ./document-crawler + docker tag breakpilot/document-crawler:${CI_COMMIT_SHA:0:8} breakpilot/document-crawler:latest + echo "Built breakpilot/document-crawler:${CI_COMMIT_SHA:0:8}" + else + echo "document-crawler Verzeichnis nicht gefunden - ueberspringe" + fi + when: + - event: tag + - event: manual + + build-admin-compliance: + image: *docker_image + commands: + - | + if [ -d ./admin-compliance ]; then + docker build -t breakpilot/admin-compliance:${CI_COMMIT_SHA:0:8} ./admin-compliance + docker tag breakpilot/admin-compliance:${CI_COMMIT_SHA:0:8} breakpilot/admin-compliance:latest + echo "Built breakpilot/admin-compliance:${CI_COMMIT_SHA:0:8}" + else + echo "admin-compliance Verzeichnis nicht gefunden - ueberspringe" + fi + when: + - event: tag + - event: manual + + build-developer-portal: + image: *docker_image + commands: + - | + if [ -d ./developer-portal ]; then + docker build -t breakpilot/developer-portal:${CI_COMMIT_SHA:0:8} ./developer-portal + docker tag breakpilot/developer-portal:${CI_COMMIT_SHA:0:8} breakpilot/developer-portal:latest + echo "Built breakpilot/developer-portal:${CI_COMMIT_SHA:0:8}" + else + echo "developer-portal Verzeichnis nicht gefunden - ueberspringe" + fi + when: + - event: tag + - event: manual + + generate-sbom: + image: *golang_image + commands: + - | + echo "Installing syft for ARM64..." + wget -qO- https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin + if [ -d ./ai-compliance-sdk ]; then + syft dir:./ai-compliance-sdk -o cyclonedx-json > sbom-ai-compliance-sdk.json + fi + for svc in backend-compliance document-crawler dsms-gateway; do + if [ -d "./$svc" ]; then + syft dir:./$svc -o cyclonedx-json > sbom-$svc.json + echo "SBOM generated for $svc" + fi + done + when: + - event: tag + - event: manual + + vulnerability-scan: + image: *golang_image + commands: + - | + echo "Installing grype for ARM64..." + wget -qO- https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + for f in sbom-*.json; do + [ -f "$f" ] || continue + echo "=== Scanning $f ===" + grype sbom:"$f" -o table --fail-on critical || true + done + when: + - event: tag + - event: manual + depends_on: + - generate-sbom + + # ======================================== + # STAGE 5: Deploy (nur manuell) + # ======================================== + + deploy-production: + image: *docker_image + commands: + - echo "Deploying breakpilot-compliance to production..." + - docker compose -f docker-compose.yml pull || true + - docker compose -f docker-compose.yml up -d --remove-orphans || true + when: + event: manual + depends_on: + - build-ai-compliance-sdk + - build-backend-compliance + - build-document-crawler + - build-admin-compliance + - build-developer-portal